From 81f389f1a3dee34536baf99d29cef03edc65fbfa Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 4 Nov 2026 14:21:34 +0100 Subject: [PATCH 01/11] build(testkit): add dependencies for the test registry and program Add linkme for the test registry, libtest-mimic and clap for the test program's command line, quick-junit for its report, and libc for process control on Unix. Refresh the lockfiles of the workspaces that depend on testkit. Signed-off-by: Evan Lezar --- tests/suites/conformance/Cargo.lock | 208 ++++++++++++++++++++++++ tests/suites/drivers/podman/Cargo.lock | 208 ++++++++++++++++++++++++ tests/suites/features/Cargo.lock | 208 ++++++++++++++++++++++++ tests/support/Cargo.lock | 208 ++++++++++++++++++++++++ tests/testkit/Cargo.lock | 209 +++++++++++++++++++++++++ tests/testkit/testkit/Cargo.toml | 14 ++ 6 files changed, 1055 insertions(+) diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index f45f1866ee..ea04cd150d 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -54,6 +54,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -382,6 +432,46 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -397,6 +487,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -679,6 +775,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + [[package]] name = "etcetera" version = "0.11.0" @@ -1395,6 +1497,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1641,6 +1749,38 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1758,6 +1898,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "newtype-uuid" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa56da58097c7ae893c4097bb6d91bb5f7fa330053e07e033349db4d5f83ba8" +dependencies = [ + "uuid", +] + [[package]] name = "nix" version = "0.29.0" @@ -1873,6 +2022,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openshell-bootstrap" version = "0.0.0" @@ -1995,11 +2150,16 @@ dependencies = [ name = "openshell-testkit" version = "0.0.0" dependencies = [ + "clap", "futures", + "libc", + "libtest-mimic", + "linkme", "openshell-bootstrap", "openshell-core", "openshell-sdk", "openshell-testkit-macros", + "quick-junit", "rand 0.9.4", "serde", "serde_json", @@ -2357,6 +2517,30 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quick-junit" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae23b8e1a59a90f7bf98caa2f4c8c2eedf6df011c6c88ed633e85a87c84526e" +dependencies = [ + "chrono", + "indexmap", + "newtype-uuid", + "quick-xml", + "strip-ansi-escapes", + "thiserror 2.0.20", + "uuid", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.45" @@ -3134,6 +3318,15 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3720,6 +3913,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -3743,6 +3942,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "want" version = "0.3.1" diff --git a/tests/suites/drivers/podman/Cargo.lock b/tests/suites/drivers/podman/Cargo.lock index 452ee563cf..6633a3e703 100644 --- a/tests/suites/drivers/podman/Cargo.lock +++ b/tests/suites/drivers/podman/Cargo.lock @@ -54,6 +54,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -382,6 +432,46 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -397,6 +487,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -678,6 +774,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + [[package]] name = "etcetera" version = "0.11.0" @@ -1394,6 +1496,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1640,6 +1748,38 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1757,6 +1897,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "newtype-uuid" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa56da58097c7ae893c4097bb6d91bb5f7fa330053e07e033349db4d5f83ba8" +dependencies = [ + "uuid", +] + [[package]] name = "nix" version = "0.29.0" @@ -1872,6 +2021,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openshell-bootstrap" version = "0.0.0" @@ -1994,11 +2149,16 @@ dependencies = [ name = "openshell-testkit" version = "0.0.0" dependencies = [ + "clap", "futures", + "libc", + "libtest-mimic", + "linkme", "openshell-bootstrap", "openshell-core", "openshell-sdk", "openshell-testkit-macros", + "quick-junit", "rand 0.9.4", "serde", "serde_json", @@ -2373,6 +2533,30 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quick-junit" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae23b8e1a59a90f7bf98caa2f4c8c2eedf6df011c6c88ed633e85a87c84526e" +dependencies = [ + "chrono", + "indexmap", + "newtype-uuid", + "quick-xml", + "strip-ansi-escapes", + "thiserror 2.0.20", + "uuid", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.45" @@ -3150,6 +3334,15 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3736,6 +3929,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -3759,6 +3958,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "want" version = "0.3.1" diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 4362c9cacd..c12c38f5ce 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -54,6 +54,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -382,6 +432,46 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -397,6 +487,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -678,6 +774,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + [[package]] name = "etcetera" version = "0.11.0" @@ -1394,6 +1496,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1658,6 +1766,38 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1775,6 +1915,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "newtype-uuid" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa56da58097c7ae893c4097bb6d91bb5f7fa330053e07e033349db4d5f83ba8" +dependencies = [ + "uuid", +] + [[package]] name = "nix" version = "0.29.0" @@ -1900,6 +2049,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openshell-bootstrap" version = "0.0.0" @@ -2034,11 +2189,16 @@ dependencies = [ name = "openshell-testkit" version = "0.0.0" dependencies = [ + "clap", "futures", + "libc", + "libtest-mimic", + "linkme", "openshell-bootstrap", "openshell-core", "openshell-sdk", "openshell-testkit-macros", + "quick-junit", "rand 0.9.4", "serde", "serde_json", @@ -2396,6 +2556,30 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quick-junit" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae23b8e1a59a90f7bf98caa2f4c8c2eedf6df011c6c88ed633e85a87c84526e" +dependencies = [ + "chrono", + "indexmap", + "newtype-uuid", + "quick-xml", + "strip-ansi-escapes", + "thiserror 2.0.20", + "uuid", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.45" @@ -3173,6 +3357,15 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3759,6 +3952,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -3782,6 +3981,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "want" version = "0.3.1" diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index a7e717b011..f8b591438f 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -54,6 +54,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -382,6 +432,46 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -397,6 +487,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -678,6 +774,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + [[package]] name = "etcetera" version = "0.11.0" @@ -1394,6 +1496,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1649,6 +1757,38 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1766,6 +1906,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "newtype-uuid" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa56da58097c7ae893c4097bb6d91bb5f7fa330053e07e033349db4d5f83ba8" +dependencies = [ + "uuid", +] + [[package]] name = "nix" version = "0.29.0" @@ -1881,6 +2030,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openshell-bootstrap" version = "0.0.0" @@ -2003,11 +2158,16 @@ dependencies = [ name = "openshell-testkit" version = "0.0.0" dependencies = [ + "clap", "futures", + "libc", + "libtest-mimic", + "linkme", "openshell-bootstrap", "openshell-core", "openshell-sdk", "openshell-testkit-macros", + "quick-junit", "rand 0.9.4", "serde", "serde_json", @@ -2373,6 +2533,30 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quick-junit" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae23b8e1a59a90f7bf98caa2f4c8c2eedf6df011c6c88ed633e85a87c84526e" +dependencies = [ + "chrono", + "indexmap", + "newtype-uuid", + "quick-xml", + "strip-ansi-escapes", + "thiserror 2.0.20", + "uuid", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.45" @@ -3150,6 +3334,15 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3736,6 +3929,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -3759,6 +3958,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "want" version = "0.3.1" diff --git a/tests/testkit/Cargo.lock b/tests/testkit/Cargo.lock index 01973c00e4..db2e79b6ca 100644 --- a/tests/testkit/Cargo.lock +++ b/tests/testkit/Cargo.lock @@ -54,6 +54,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -382,6 +432,46 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -397,6 +487,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -669,6 +765,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + [[package]] name = "etcetera" version = "0.11.0" @@ -1385,6 +1487,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1631,6 +1739,38 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1748,6 +1888,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "newtype-uuid" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa56da58097c7ae893c4097bb6d91bb5f7fa330053e07e033349db4d5f83ba8" +dependencies = [ + "uuid", +] + [[package]] name = "nix" version = "0.29.0" @@ -1863,6 +2012,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openshell-bootstrap" version = "0.0.0" @@ -1985,14 +2140,20 @@ dependencies = [ name = "openshell-testkit" version = "0.0.0" dependencies = [ + "clap", "futures", + "libc", + "libtest-mimic", + "linkme", "openshell-bootstrap", "openshell-core", "openshell-sdk", "openshell-testkit-macros", + "quick-junit", "rand 0.9.4", "serde", "serde_json", + "tempfile", "tokio", "tonic", ] @@ -2348,6 +2509,30 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quick-junit" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae23b8e1a59a90f7bf98caa2f4c8c2eedf6df011c6c88ed633e85a87c84526e" +dependencies = [ + "chrono", + "indexmap", + "newtype-uuid", + "quick-xml", + "strip-ansi-escapes", + "thiserror 2.0.20", + "uuid", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.45" @@ -3125,6 +3310,15 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3711,6 +3905,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -3734,6 +3934,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "want" version = "0.3.1" diff --git a/tests/testkit/testkit/Cargo.toml b/tests/testkit/testkit/Cargo.toml index b8570fbe2c..47ff56c13c 100644 --- a/tests/testkit/testkit/Cargo.toml +++ b/tests/testkit/testkit/Cargo.toml @@ -19,11 +19,25 @@ openshell-bootstrap = { path = "../../../crates/openshell-bootstrap" } openshell-core = { path = "../../../crates/openshell-core", default-features = false } openshell-sdk = { path = "../../../crates/openshell-sdk" } openshell-testkit-macros = { path = "../testkit-macros" } +clap = { version = "4", features = ["derive"] } futures = "0.3" +libtest-mimic = "0.8" +linkme = "0.3" +quick-junit = "0.8" rand = "0.9" serde = "1" serde_json = "1" tokio = { version = "1.43", features = ["macros", "process", "rt", "time"] } +[target.'cfg(unix)'.dependencies] +libc = "0.2" + [dev-dependencies] +tempfile = "3" tonic = "0.14" + +# A test program, run by `cargo test` and nextest through `testkit::main`'s +# libtest interface, that checks the program's own subcommands. +[[test]] +name = "program" +harness = false From ad9bde286531a6dc791d18a63477238f3899268e Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 4 Nov 2026 14:23:26 +0100 Subject: [PATCH 02/11] feat(testkit): register tests and add a test program with list `#[testkit::test]` now also registers each test, with its family, name, description and `#[ignore]` reason, in a link-time registry. The hidden function gets only `cfg` and lint attributes; `#[should_panic]`, `#[cfg_attr]` and `#[expect]` are rejected with an error that says why. `testkit::main()` turns a binary into a test program. It takes libtest's arguments, so it also works as a `harness = false` target under cargo test and nextest, and it has a `list` subcommand that prints each test's name and description. A program that links no tests exits 2, since Rust does not link a suite crate that is never named. Signed-off-by: Evan Lezar --- tests/testkit/testkit-macros/src/lib.rs | 212 ++++++++++++++++--- tests/testkit/testkit/src/lib.rs | 10 + tests/testkit/testkit/src/program/mod.rs | 198 ++++++++++++++++++ tests/testkit/testkit/src/registry.rs | 208 ++++++++++++++++++ tests/testkit/testkit/tests/attribute.rs | 116 ++++++++++ tests/testkit/testkit/tests/program.rs | 256 +++++++++++++++++++++++ 6 files changed, 973 insertions(+), 27 deletions(-) create mode 100644 tests/testkit/testkit/src/program/mod.rs create mode 100644 tests/testkit/testkit/src/registry.rs create mode 100644 tests/testkit/testkit/tests/attribute.rs create mode 100644 tests/testkit/testkit/tests/program.rs diff --git a/tests/testkit/testkit-macros/src/lib.rs b/tests/testkit/testkit-macros/src/lib.rs index fc3dbbdb68..b8a614f3c6 100644 --- a/tests/testkit/testkit-macros/src/lib.rs +++ b/tests/testkit/testkit-macros/src/lib.rs @@ -13,12 +13,28 @@ //! async fn expose_service(cx: &mut Context) -> testkit::Result { /* ... */ } //! ``` //! -//! The attribute expands to a `#[tokio::test]` wrapper with the same name, so -//! the nextest test name is the function name. The original function is nested -//! in the wrapper and passed to `testkit::run`. Other attributes, such as -//! `#[ignore]` or `#[cfg(...)]`, move to the wrapper. `#[should_panic]` is -//! rejected: a testkit test panics on any failure, so it would pass for the -//! wrong reason. Check the expected failure in the body instead. +//! The attribute generates three items: +//! +//! - a hidden `async fn` that runs the original function through +//! `testkit::run`, with the original function nested in it; +//! - under `#[cfg(test)]`, a `#[tokio::test]` wrapper with the same name that +//! calls it, so `cargo test` and nextest run the test like any other and the +//! test name is the function name; +//! - an entry in `testkit`'s test registry, with the test's family, name, +//! description, and `#[ignore]`, which `testkit::main()` runs. +//! +//! Other attributes, such as `#[ignore]` or `#[cfg(...)]`, move to the +//! wrapper. `#[cfg(...)]` also applies to the other two items, and lint +//! attributes (`allow`, `warn`, `deny`, `forbid`) also to the hidden +//! function. `#[should_panic]` is rejected: a testkit test panics on any +//! failure, so it would pass for the wrong reason. Check the expected failure +//! in the body instead. `#[cfg_attr]` and `#[expect]` are rejected too. +//! +//! A test's family is the first directory under `tests/suites/` in its +//! crate's path: `conformance` for `tests/suites/conformance`, `features` for +//! `tests/suites/features/`. + +use std::path::Path; use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; @@ -27,12 +43,17 @@ use syn::{Attribute, Expr, ExprLit, ItemFn, Lit, LitStr, Meta}; #[proc_macro_attribute] pub fn test(attr: TokenStream, item: TokenStream) -> TokenStream { - expand(attr.into(), item.into()) + let family = std::env::var_os("CARGO_MANIFEST_DIR").and_then(|dir| family_of(Path::new(&dir))); + expand(attr.into(), item.into(), family.as_deref()) .unwrap_or_else(syn::Error::into_compile_error) .into() } -fn expand(attr: TokenStream2, item: TokenStream2) -> syn::Result { +fn expand( + attr: TokenStream2, + item: TokenStream2, + family: Option<&str>, +) -> syn::Result { let mut body: ItemFn = syn::parse2(item)?; if !attr.is_empty() { return Err(syn::Error::new_spanned( @@ -58,17 +79,24 @@ fn expand(attr: TokenStream2, item: TokenStream2) -> syn::Result { "`#[testkit::test]` already makes this a test; remove this attribute", )); } - if let Some(attr) = body - .attrs - .iter() - .find(|attr| attr.path().is_ident("should_panic")) - { - return Err(syn::Error::new_spanned( - attr, + for attr in &body.attrs { + let message = if attr.path().is_ident("should_panic") { "testkit tests panic on any failure, so `#[should_panic]` would pass for the \ wrong reason; check the expected failure in the body, for example with \ - `CliOutput::failure` and `testkit::ensure!`", - )); + `CliOutput::failure` and `testkit::ensure!`" + } else if attr.path().is_ident("cfg_attr") { + // The registration reads `#[ignore]` and `#[cfg]` directly, so it + // would not see them inside `cfg_attr`. + "`#[cfg_attr]` is not supported on testkit tests; use `#[cfg]` or `#[ignore]` \ + directly" + } else if attr.path().is_ident("expect") { + // The lint attributes are copied to two functions, and only one + // of them would raise the lint. + "`#[expect]` is not supported on testkit tests; use `#[allow]`" + } else { + continue; + }; + return Err(syn::Error::new_spanned(attr, message)); } let description = doc_comment(&body.attrs); if description.is_empty() { @@ -77,20 +105,46 @@ fn expand(attr: TokenStream2, item: TokenStream2) -> syn::Result { "testkit tests need a doc comment describing the behavior they verify", )); } + let ignore = ignore(&body.attrs)?; let name = body.sig.ident.clone(); let name_str = LitStr::new(&name.to_string(), name.span()); let description = LitStr::new(&description, name.span()); + let family = match family { + Some(family) => quote!(::core::option::Option::Some(#family)), + None => quote!(::core::option::Option::None), + }; + let (ignored, ignore_reason) = match ignore { + None => (false, quote!(::core::option::Option::None)), + Some(None) => (true, quote!(::core::option::Option::None)), + Some(Some(reason)) => (true, quote!(::core::option::Option::Some(#reason))), + }; let vis = body.vis.clone(); let attrs = std::mem::take(&mut body.attrs); + let cfgs: Vec<&Attribute> = attrs + .iter() + .filter(|attr| attr.path().is_ident("cfg")) + .collect(); + // `#[cfg]` and lint attributes such as `#[allow(...)]` must reach the + // body, which is nested in the hidden function. Anything else is for the + // test itself, so goes on the wrapper only. + let body_attrs: Vec<&Attribute> = attrs + .iter() + .filter(|attr| { + ["cfg", "allow", "warn", "deny", "forbid"] + .iter() + .any(|name| attr.path().is_ident(name)) + }) + .collect(); body.vis = syn::Visibility::Inherited; body.sig.ident = format_ident!("body"); + let entry = format_ident!("__testkit_{}", name); Ok(quote! { - #(#attrs)* - #[::testkit::__private::tokio::test(crate = "::testkit::__private::tokio")] - #vis async fn #name() { + #(#body_attrs)* + #[doc(hidden)] + async fn #entry() { #body ::testkit::run( ::testkit::TestInfo { @@ -101,6 +155,30 @@ fn expand(attr: TokenStream2, item: TokenStream2) -> syn::Result { ) .await; } + + #(#attrs)* + #[cfg(test)] + #[::testkit::__private::tokio::test(crate = "::testkit::__private::tokio")] + #vis async fn #name() { + #entry().await; + } + + #(#cfgs)* + const _: () = { + #[::testkit::__private::linkme::distributed_slice(::testkit::__private::TESTS)] + #[linkme(crate = ::testkit::__private::linkme)] + static TEST: ::testkit::Test = ::testkit::Test { + info: ::testkit::TestInfo { + name: #name_str, + description: #description, + }, + family: #family, + module_path: ::core::module_path!(), + ignored: #ignored, + ignore_reason: #ignore_reason, + run: || ::std::boxed::Box::pin(#entry()), + }; + }; }) } @@ -112,6 +190,31 @@ fn is_test_attr(attr: &Attribute) -> bool { .is_some_and(|segment| segment.ident == "test") } +/// `None` if the test is not ignored, else the `#[ignore = "..."]` reason if +/// it has one. +fn ignore(attrs: &[Attribute]) -> syn::Result>> { + let Some(attr) = attrs.iter().find(|attr| attr.path().is_ident("ignore")) else { + return Ok(None); + }; + match &attr.meta { + Meta::Path(_) => Ok(Some(None)), + Meta::NameValue(meta) => match &meta.value { + Expr::Lit(ExprLit { + lit: Lit::Str(reason), + .. + }) => Ok(Some(Some(reason.clone()))), + value => Err(syn::Error::new_spanned( + value, + "the `#[ignore]` reason must be a string literal", + )), + }, + Meta::List(_) => Err(syn::Error::new_spanned( + attr, + "use `#[ignore]` or `#[ignore = \"reason\"]`", + )), + } +} + /// The doc comment, joined into one line. Empty if there is none. fn doc_comment(attrs: &[Attribute]) -> String { attrs @@ -131,21 +234,40 @@ fn doc_comment(attrs: &[Attribute]) -> String { .join(" ") } +/// The family of a suite crate in `manifest_dir`: the first directory under +/// the nearest enclosing `suites/`. +fn family_of(manifest_dir: &Path) -> Option { + let components: Vec<_> = manifest_dir + .components() + .filter_map(|component| component.as_os_str().to_str()) + .collect(); + let suites = components + .iter() + .rposition(|component| *component == "suites")?; + components + .get(suites + 1) + .map(|family| (*family).to_string()) +} + #[cfg(test)] mod tests { - use super::expand; + use std::path::Path; + + use super::{expand, family_of}; + + const OK_FN: &str = "async fn t(cx: &mut Context) -> testkit::Result { Ok(()) }"; // A suite only compiles correct tests, so it never reaches these errors. + // `testkit/tests/macro.rs` checks what correct tests expand to. #[test] fn rejects_misuse() { - let ok_fn = "async fn t(cx: &mut Context) -> testkit::Result { Ok(()) }"; for (attr, item, error) in [ ( "id = \"t-1\"", - format!("/// Doc.\n{ok_fn}"), + format!("/// Doc.\n{OK_FN}"), "takes no arguments", ), - ("", ok_fn.to_string(), "doc comment"), + ("", OK_FN.to_string(), "doc comment"), ( "", "/// Doc.\nfn t(cx: &mut Context) -> testkit::Result { Ok(()) }".to_string(), @@ -158,17 +280,53 @@ mod tests { ), ( "", - format!("/// Doc.\n#[tokio::test]\n{ok_fn}"), + format!("/// Doc.\n#[tokio::test]\n{OK_FN}"), "already makes this a test", ), ( "", - format!("/// Doc.\n#[should_panic]\n{ok_fn}"), + format!("/// Doc.\n#[should_panic]\n{OK_FN}"), "`#[should_panic]`", ), + ( + "", + format!("/// Doc.\n#[ignore(later)]\n{OK_FN}"), + "`#[ignore]` or", + ), + ( + "", + format!("/// Doc.\n#[ignore = 1]\n{OK_FN}"), + "string literal", + ), + ( + "", + format!("/// Doc.\n#[cfg_attr(unix, ignore)]\n{OK_FN}"), + "`#[cfg_attr]` is not supported", + ), + ( + "", + format!("/// Doc.\n#[expect(unused)]\n{OK_FN}"), + "`#[expect]` is not supported", + ), ] { - let err = expand(attr.parse().unwrap(), item.parse().unwrap()).unwrap_err(); + let err = expand(attr.parse().unwrap(), item.parse().unwrap(), None).unwrap_err(); assert!(err.to_string().contains(error), "{item}: {err}"); } } + + #[test] + fn family_is_the_directory_under_suites() { + for (dir, family) in [ + ("/repo/tests/suites/conformance", Some("conformance")), + ( + "/repo/tests/suites/features/provider-refresh/keycloak", + Some("features"), + ), + ("/a/suites/x/tests/suites/drivers/podman", Some("drivers")), + ("/repo/tests/testkit/testkit", None), + ("/repo/tests/suites", None), + ] { + assert_eq!(family_of(Path::new(dir)).as_deref(), family, "{dir}"); + } + } } diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 10db7d0e33..4f5148b1a5 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -21,6 +21,10 @@ //! //! Tests find the gateway the same way the `openshell` CLI does (see //! [`gateway`]). +//! +//! `#[testkit::test]` also registers each test, so a binary that calls +//! [`main`] is a test program that lists, runs, and reports its tests without +//! Cargo (see [`program`]). mod cleanup; mod cli; @@ -29,6 +33,8 @@ mod error; pub mod gateway; mod image; mod poll; +pub mod program; +mod registry; mod run; pub use cleanup::CLEANUP_TIMEOUT; @@ -42,10 +48,14 @@ pub use gateway::Gateway; pub use image::{BASE_IMAGE_ENV, DEFAULT_BASE_IMAGE, base_image}; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until}; +pub use program::main; +pub use registry::Test; pub use run::{TestInfo, run}; pub use testkit_macros::test; #[doc(hidden)] pub mod __private { + pub use crate::registry::TESTS; + pub use linkme; pub use tokio; } diff --git a/tests/testkit/testkit/src/program/mod.rs b/tests/testkit/testkit/src/program/mod.rs new file mode 100644 index 0000000000..0216a6ad11 --- /dev/null +++ b/tests/testkit/testkit/src/program/mod.rs @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `testkit::main()`: a test program anyone can run against their gateway. +//! +//! A program is a binary whose `main` calls [`main`], linked with the crates +//! holding its `#[testkit::test]`s: +//! +//! ```ignore +//! fn main() -> std::process::ExitCode { +//! testkit::main() +//! } +//! ``` +//! +//! It has one subcommand so far: `list [FILTER]...` prints each test's name +//! and description. +//! +//! Any other arguments are libtest's, so the program also works as a +//! `harness = false` test target under `cargo test` and nextest: it answers +//! `--list --format terse`, `--ignored`, and ` --exact --nocapture`. +//! In that mode tests run in the program's own process, one at a time unless +//! `--test-threads` says otherwise. + +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::{Parser, Subcommand}; + +use crate::registry::{RESERVED_NAMES, Selection, TESTS, Test, validate}; + +/// Exit code when tests could not run, including usage errors. +const ERROR_EXIT: u8 = 2; +/// Set by `run` for the test processes it starts. +const SUPERVISED_ENV: &str = "TESTKIT_SUPERVISED"; + +/// Run the program: the subcommands, or libtest's interface. +pub fn main() -> ExitCode { + main_with(&TESTS) +} + +/// [`main`] over `tests` instead of every registered test. +#[doc(hidden)] +pub fn main_with(tests: &'static [Test]) -> ExitCode { + let args: Vec = std::env::args_os().collect(); + let program = program_name(&args); + // A suite crate that the program never names is not linked, and its + // tests are silently missing. Fail instead of passing with nothing. + if tests.is_empty() { + eprintln!( + "{program}: no tests are linked into this program; it must name each suite crate \ + it runs, for example with `use as _;`" + ); + return ExitCode::from(ERROR_EXIT); + } + if let Err(error) = validate(tests) { + eprintln!("{program}: {error}"); + return ExitCode::from(ERROR_EXIT); + } + let first = args.get(1).and_then(|arg| arg.to_str()); + if first.is_some_and(|arg| RESERVED_NAMES.contains(&arg) || arg == "-h" || arg == "--help") { + let cli = match Cli::try_parse_from(&args) { + Ok(cli) => cli, + Err(error) => { + let _ = error.print(); + return ExitCode::from(if error.use_stderr() { ERROR_EXIT } else { 0 }); + } + }; + command(&program, tests, cli.command) + } else { + libtest(tests, args) + } +} + +#[derive(Parser)] +#[command( + about = "Run OpenShell tests against a gateway", + after_help = "The gateway is the one the openshell CLI selects: OPENSHELL_GATEWAY_ENDPOINT, \ + OPENSHELL_GATEWAY, or the active gateway. OPENSHELL_WORKSPACE selects the \ + workspace and OPENSHELL_BIN the CLI. Only plaintext and mTLS gateways are \ + supported.\n\nWithout a subcommand the program takes libtest's arguments, \ + as under cargo test and nextest." +)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Print each test's name and description, including ignored tests. + List { + /// Show tests whose name contains any of these (all if none). + filters: Vec, + }, +} + +fn command(_program: &str, tests: &'static [Test], command: Command) -> ExitCode { + match command { + Command::List { filters } => { + list(tests, filters); + ExitCode::SUCCESS + } + } +} + +/// One line per test: name and description, in aligned columns. +/// An ignored test's description ends with `[ignored]` or its reason. +fn list(tests: &[Test], filters: Vec) { + let selection = Selection { + filters, + include_ignored: true, + }; + let rows: Vec<_> = selection + .apply(tests) + .into_iter() + .map(|(test, _)| (test.name(), describe(test))) + .collect(); + let name_width = rows.iter().map(|row| row.0.len()).max().unwrap_or(0); + for (name, description) in rows { + println!("{name:name_width$} {description}"); + } +} + +fn describe(test: &Test) -> String { + let description = test.info.description; + match (test.ignored, test.ignore_reason) { + (false, _) => description.to_string(), + (true, None) => format!("{description} [ignored]"), + (true, Some(reason)) => format!("{description} [ignored: {reason}]"), + } +} + +/// libtest's interface, for `cargo test` and nextest, and for `run`'s test +/// processes. +fn libtest(tests: &'static [Test], args: Vec) -> ExitCode { + let mut args = libtest_mimic::Arguments::from_iter(args); + // Tests share one gateway; run them one at a time unless asked otherwise. + if args.test_threads.is_none() { + args.test_threads = Some(1); + } + let mut tests: Vec<&'static Test> = tests.iter().collect(); + tests.sort_by_cached_key(|test| test.name()); + let trials = tests + .into_iter() + .map(|test| { + libtest_mimic::Trial::test(test.name(), move || { + run_in_process(test); + Ok(()) + }) + .with_ignored_flag(test.ignored) + }) + .collect(); + let conclusion = libtest_mimic::run(&args, trials); + // `run` starts this process for exactly one test. Running none would + // look like a pass. + if std::env::var_os(SUPERVISED_ENV).is_some() + && !args.list + && conclusion.num_passed + conclusion.num_failed != 1 + { + eprintln!( + "testkit: expected to run exactly one test, ran {}", + conclusion.num_passed + conclusion.num_failed + ); + return ExitCode::from(ERROR_EXIT); + } + conclusion.exit_code() +} + +/// Run one test on a fresh current-thread runtime, as `#[tokio::test]` does. +/// Panics to fail it. +fn run_in_process(test: &Test) { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("start the async runtime") + .block_on((test.run)()); +} + +fn program_name(args: &[OsString]) -> String { + args.first() + .map(PathBuf::from) + .and_then(|path| { + path.file_name() + .map(|name| name.to_string_lossy().into_owned()) + }) + .unwrap_or_else(|| "testkit".to_string()) +} + +#[cfg(test)] +mod tests { + use super::Cli; + + #[test] + fn cli_is_consistent() { + ::command().debug_assert(); + } +} diff --git a/tests/testkit/testkit/src/registry.rs b/tests/testkit/testkit/src/registry.rs new file mode 100644 index 0000000000..aceaed673f --- /dev/null +++ b/tests/testkit/testkit/src/registry.rs @@ -0,0 +1,208 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The tests linked into a program, registered by `#[testkit::test]`. + +use std::collections::BTreeSet; +use std::future::Future; +use std::pin::Pin; + +use crate::run::TestInfo; + +/// Every `#[testkit::test]` linked into the program. +#[linkme::distributed_slice] +pub static TESTS: [Test]; + +/// A registered test. +#[derive(Debug)] +pub struct Test { + pub info: TestInfo, + /// The directory under `tests/suites/` holding the test's crate, if any. + pub family: Option<&'static str>, + /// `module_path!()` where the test is defined. + pub module_path: &'static str, + pub ignored: bool, + /// The reason given in `#[ignore = "..."]`. + pub ignore_reason: Option<&'static str>, + /// Runs the test through [`crate::run`]; panics to fail it. + pub run: fn() -> Pin>>, +} + +impl Test { + /// The test's name, as libtest and nextest name it: the module path + /// without the crate, then the function name. + pub fn name(&self) -> String { + match self.module_path.split_once("::") { + Some((_crate, module)) => format!("{module}::{}", self.info.name), + None => self.info.name.to_string(), + } + } + + pub fn family(&self) -> &'static str { + self.family.unwrap_or("-") + } +} + +/// Names a test cannot have at the top level: the program would read them as +/// its own subcommands, so nextest could not run the test by name. +pub const RESERVED_NAMES: &[&str] = &["help", "list", "ready", "run"]; + +/// Check the tests can be told apart by name. +pub fn validate(tests: &[Test]) -> Result<(), String> { + let mut names = BTreeSet::new(); + for test in tests { + let name = test.name(); + if RESERVED_NAMES.contains(&name.as_str()) { + return Err(format!( + "test {name} has the name of a subcommand; rename it or move it into a module" + )); + } + if !names.insert(name.clone()) { + return Err(format!( + "two tests are named {name}; test names must be unique in a program" + )); + } + } + Ok(()) +} + +/// Which tests to run. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Selection { + /// Run tests whose name contains any of these; all tests if empty. + pub filters: Vec, + /// Run `#[ignore]`d tests too, instead of reporting them as skipped. + pub include_ignored: bool, +} + +/// What a selection does with one test. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Disposition { + Run, + /// Selected, but ignored and not asked for. + Skip, +} + +impl Selection { + /// The tests this selects, in name order, with what to do with each. + pub fn apply<'a>(&self, tests: &'a [Test]) -> Vec<(&'a Test, Disposition)> { + let mut selected: Vec<_> = tests + .iter() + .filter_map(|test| { + self.disposition(test) + .map(|disposition| (test, disposition)) + }) + .collect(); + selected.sort_by_cached_key(|(test, _)| test.name()); + selected + } + + fn disposition(&self, test: &Test) -> Option { + let name = test.name(); + if !self.filters.is_empty() + && !self + .filters + .iter() + .any(|filter| name.contains(filter.as_str())) + { + return None; + } + if test.ignored && !self.include_ignored { + Some(Disposition::Skip) + } else { + Some(Disposition::Run) + } + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::{Disposition, Selection, Test, validate}; + use crate::run::TestInfo; + + pub(crate) const fn test(module_path: &'static str, name: &'static str, ignored: bool) -> Test { + Test { + info: TestInfo { + name, + description: "A test.", + }, + family: Some("conformance"), + module_path, + ignored, + ignore_reason: None, + run: || Box::pin(async {}), + } + } + + const TESTS: &[Test] = &[ + test("sandbox::smoke", "create_exec_delete", false), + test("sandbox::lifecycle", "stop_start", false), + test("sandbox::lifecycle", "restart_later", true), + test("smoke", "top_level", false), + ]; + + fn names(selection: &Selection) -> Vec<(String, Disposition)> { + selection + .apply(TESTS) + .into_iter() + .map(|(test, disposition)| (test.name(), disposition)) + .collect() + } + + #[test] + fn names_follow_libtest() { + assert_eq!(TESTS[0].name(), "smoke::create_exec_delete"); + assert_eq!(TESTS[3].name(), "top_level"); + } + + #[test] + fn selection_follows_libtest() { + use Disposition::{Run, Skip}; + let all = Selection::default(); + assert_eq!( + names(&all), + [ + ("lifecycle::restart_later".into(), Skip), + ("lifecycle::stop_start".into(), Run), + ("smoke::create_exec_delete".into(), Run), + ("top_level".into(), Run), + ] + ); + + let filtered = Selection { + filters: vec!["stop".into(), "top".into()], + ..Selection::default() + }; + assert_eq!( + names(&filtered), + [ + ("lifecycle::stop_start".into(), Run), + ("top_level".into(), Run) + ] + ); + + let include = Selection { + filters: vec!["lifecycle".into()], + include_ignored: true, + }; + assert_eq!( + names(&include), + [ + ("lifecycle::restart_later".into(), Run), + ("lifecycle::stop_start".into(), Run) + ] + ); + } + + #[test] + fn validate_rejects_ambiguous_names() { + assert!(validate(TESTS).is_ok()); + let duplicate = [test("a::m", "t", false), test("b::m", "t", false)]; + let error = validate(&duplicate).unwrap_err(); + assert!(error.contains("two tests are named m::t"), "{error}"); + let reserved = [test("suite", "run", false)]; + let error = validate(&reserved).unwrap_err(); + assert!(error.contains("name of a subcommand"), "{error}"); + assert!(validate(&[test("suite::m", "run", false)]).is_ok()); + } +} diff --git a/tests/testkit/testkit/tests/attribute.rs b/tests/testkit/testkit/tests/attribute.rs new file mode 100644 index 0000000000..1f49adea40 --- /dev/null +++ b/tests/testkit/testkit/tests/attribute.rs @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! What `#[testkit::test]` expands to, checked by compiling tests with it in +//! an ordinary libtest target. The tests need a gateway, so they are ignored; +//! the check compares the registry with what libtest lists. + +use std::collections::BTreeSet; +use std::process::Command; + +/// Ignored with a reason. +#[testkit::test] +#[ignore = "needs a gateway"] +async fn ignored_with_reason(_cx: &mut testkit::Context) -> testkit::Result { + Ok(()) +} + +/// Lint attributes reach the body: `cx` is unused. +#[testkit::test] +#[ignore] +#[allow(unused_variables)] +async fn lint_allowed(cx: &mut testkit::Context) -> testkit::Result { + Ok(()) +} + +/// Compiled out, so neither a test nor registered. +#[testkit::test] +#[cfg(any())] +async fn compiled_out(_cx: &mut testkit::Context) -> testkit::Result { + Ok(()) +} + +mod nested { + /// Named with its module. + #[testkit::test] + #[ignore] + async fn in_a_module(_cx: &mut testkit::Context) -> testkit::Result { + Ok(()) + } +} + +/// The names libtest lists for this binary, with `extra` arguments. +fn libtest_list(extra: &[&str]) -> BTreeSet { + let output = Command::new(std::env::current_exe().expect("current exe")) + .args(["--list", "--format", "terse"]) + .args(extra) + .output() + .expect("list the tests"); + assert!(output.status.success(), "{output:?}"); + String::from_utf8(output.stdout) + .expect("utf-8") + .lines() + .filter_map(|line| line.strip_suffix(": test")) + .filter(|name| *name != "registry_matches_libtest") + .map(str::to_string) + .collect() +} + +#[test] +fn registry_matches_libtest() { + let registered: Vec<_> = testkit::__private::TESTS + .iter() + .map(|test| { + ( + test.name(), + test.module_path, + test.family, + test.ignored, + test.ignore_reason, + test.info.description, + ) + }) + .collect::>() + .into_iter() + .collect(); + assert_eq!( + registered, + [ + ( + "ignored_with_reason".to_string(), + "attribute", + None, + true, + Some("needs a gateway"), + "Ignored with a reason." + ), + ( + "lint_allowed".to_string(), + "attribute", + None, + true, + None, + "Lint attributes reach the body: `cx` is unused." + ), + ( + "nested::in_a_module".to_string(), + "attribute::nested", + None, + true, + None, + "Named with its module." + ), + ] + ); + + // libtest runs and ignores the same tests, under the same names. + let names = |ignored_only: bool| -> BTreeSet { + testkit::__private::TESTS + .iter() + .filter(|test| test.ignored || !ignored_only) + .map(testkit::Test::name) + .collect() + }; + assert_eq!(libtest_list(&[]), names(false)); + assert_eq!(libtest_list(&["--ignored"]), names(true)); +} diff --git a/tests/testkit/testkit/tests/program.rs b/tests/testkit/testkit/tests/program.rs new file mode 100644 index 0000000000..16496c0b69 --- /dev/null +++ b/tests/testkit/testkit/tests/program.rs @@ -0,0 +1,256 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Checks of `testkit::main()`, as a test program. +//! +//! This binary is three programs, chosen by `TESTKIT_FIXTURE`: +//! +//! - unset: the checks below, run by `cargo test` and nextest through the +//! program's libtest interface; +//! - `fixtures`: tests that pass, fail, hang, ignore `SIGTERM`, or leave +//! processes behind, which the checks run through the program's +//! subcommands; +//! - `registered`: `testkit::main()` over the `#[testkit::test]`s below; +//! - `empty`: a program with no tests. + +#[cfg(unix)] +fn main() -> std::process::ExitCode { + match std::env::var("TESTKIT_FIXTURE").as_deref() { + Ok("fixtures") => testkit::program::main_with(&unix::FIXTURES), + Ok("registered") => testkit::main(), + Ok("empty") => testkit::program::main_with(&[]), + _ => testkit::program::main_with(&unix::CHECKS), + } +} + +#[cfg(not(unix))] +fn main() {} + +/// Needs a gateway. +#[testkit::test] +async fn registered_needs_a_gateway(cx: &mut testkit::Context) -> testkit::Result { + cx.cli(["status"]).await?.success()?; + Ok(()) +} + +/// Ignored until later. +#[testkit::test] +#[ignore = "https://github.com/NVIDIA/OpenShell/issues/1"] +async fn registered_ignored(_cx: &mut testkit::Context) -> testkit::Result { + Ok(()) +} + +#[cfg(unix)] +mod unix { + use std::future::Future; + use std::os::unix::process::CommandExt as _; + use std::pin::Pin; + use std::process::{Command, Output, Stdio}; + use std::time::Duration; + + use testkit::{Test, TestInfo}; + + const PID_FILE_ENV: &str = "FIXTURE_PID_FILE"; + + type Run = fn() -> Pin>>; + + const fn test(module_path: &'static str, name: &'static str, run: Run) -> Test { + Test { + info: TestInfo { + name, + description: "A fixture.", + }, + family: Some("fixtures"), + module_path, + ignored: false, + ignore_reason: None, + run, + } + } + + const fn fixture(name: &'static str, run: Run) -> Test { + test("program::fixture", name, run) + } + + const fn check(name: &'static str, run: Run) -> Test { + test("program::check", name, run) + } + + fn spawn_helper_and_record_its_pid() { + // A helper in the test's process group, like a CLI process. It is + // never waited on: the supervisor must kill it. + #[allow(clippy::zombie_processes)] + let helper = Command::new("sleep") + .arg("300") + .spawn() + .expect("spawn sleep"); + let pid_file = std::env::var(PID_FILE_ENV).expect("pid file"); + std::fs::write(pid_file, helper.id().to_string()).expect("write pid"); + } + + pub static FIXTURES: [Test; 7] = [ + fixture("passes", || { + Box::pin(async { + println!("passes: stdout"); + eprintln!("passes: stderr"); + }) + }), + fixture("fails", || Box::pin(async { panic!("fixture failure") })), + fixture("hangs", || { + Box::pin(async { + spawn_helper_and_record_its_pid(); + std::thread::sleep(Duration::from_secs(300)); + }) + }), + fixture("leaves_a_helper", || { + Box::pin(async { spawn_helper_and_record_its_pid() }) + }), + fixture("interrupts_after_exit", || { + Box::pin(async { + // Signal the supervisor once this test has exited, while it + // waits for the test's output to close. The helper leaves + // the test's process group so that it survives the test, and + // holds the output open until it has sent the signal. + let supervisor = std::os::unix::process::parent_id(); + #[allow(clippy::zombie_processes)] + let _signaller = Command::new("sh") + .args(["-c", &format!("sleep 1; kill -TERM {supervisor}")]) + .process_group(0) + .spawn() + .expect("spawn the signaller"); + }) + }), + fixture("ignores_sigterm", || { + Box::pin(async { + // SAFETY: plain syscall. + unsafe { + libc::signal(libc::SIGTERM, libc::SIG_IGN); + } + std::thread::sleep(Duration::from_secs(300)); + }) + }), + Test { + ignored: true, + ignore_reason: Some("not today"), + ..fixture("ignored", || Box::pin(async {})) + }, + ]; + + pub static CHECKS: [Test; 3] = [ + check("nextest_interface", || { + Box::pin(async { nextest_interface() }) + }), + check("list_shows_name_and_description", || { + Box::pin(async { list_shows_name_and_description() }) + }), + check("a_program_without_tests_fails", || { + Box::pin(async { a_program_without_tests_fails() }) + }), + ]; + + fn program(fixture: &str, args: &[&str]) -> Command { + let mut command = Command::new(std::env::current_exe().expect("current exe")); + command + .args(args) + .env("TESTKIT_FIXTURE", fixture) + .env_remove("TESTKIT_SUPERVISED") + .stdin(Stdio::null()); + command + } + + fn output(mut command: Command) -> Output { + let output = command.output().expect("run the program"); + eprintln!( + "{command:?}: {}\n--- stdout\n{}--- stderr\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + output + } + + fn stdout(output: &Output) -> &str { + std::str::from_utf8(&output.stdout).expect("utf-8") + } + + fn stderr(output: &Output) -> &str { + std::str::from_utf8(&output.stderr).expect("utf-8") + } + + /// nextest lists with `--list --format terse [--ignored]` and runs each + /// test with ` --exact --nocapture`. + fn nextest_interface() { + let list = output(program("fixtures", &["--list", "--format", "terse"])); + assert!(list.status.success()); + assert_eq!( + stdout(&list), + "fixture::fails: test\nfixture::hangs: test\nfixture::ignored: test\n\ + fixture::ignores_sigterm: test\nfixture::interrupts_after_exit: test\n\ + fixture::leaves_a_helper: test\nfixture::passes: test\n" + ); + let ignored = output(program( + "fixtures", + &["--list", "--format", "terse", "--ignored"], + )); + assert_eq!(stdout(&ignored), "fixture::ignored: test\n"); + + let pass = output(program( + "fixtures", + &["fixture::passes", "--exact", "--nocapture"], + )); + assert!(pass.status.success()); + assert!(stdout(&pass).contains("passes: stdout")); + assert!(stdout(&pass).contains("1 passed")); + + let fail = output(program( + "fixtures", + &["fixture::fails", "--exact", "--nocapture"], + )); + assert_eq!(fail.status.code(), Some(101)); + assert!(stderr(&fail).contains("fixture failure")); + + // An ignored test runs only when asked for. + let skip = output(program("fixtures", &["fixture::ignored", "--exact"])); + assert!(stdout(&skip).contains("1 ignored")); + let run = output(program( + "fixtures", + &["fixture::ignored", "--exact", "--ignored"], + )); + assert!(stdout(&run).contains("1 passed")); + } + + fn list_shows_name_and_description() { + let list = output(program("registered", &["list"])); + assert!(list.status.success()); + let lines: Vec<_> = stdout(&list).lines().collect(); + assert_eq!(lines.len(), 2, "{lines:?}"); + assert_eq!( + lines[0], + "registered_ignored Ignored until later. \ + [ignored: https://github.com/NVIDIA/OpenShell/issues/1]" + ); + assert_eq!(lines[1], "registered_needs_a_gateway Needs a gateway."); + + let filtered = output(program("fixtures", &["list", "pass"])); + assert_eq!(stdout(&filtered), "fixture::passes A fixture.\n"); + } + + /// A program whose suites were not linked fails rather than passing with + /// no tests, under every interface. + fn a_program_without_tests_fails() { + for args in [ + &["list"][..], + &["run"], + &["--list", "--format", "terse"], + &[], + ] { + let empty = output(program("empty", args)); + assert_eq!(empty.status.code(), Some(2), "{args:?}"); + assert!( + stderr(&empty).contains("no tests are linked into this program"), + "{args:?}" + ); + } + } + +} From d6f373585f08e004ef1bccbd04700ce3c84727b8 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 4 Nov 2026 14:24:09 +0100 Subject: [PATCH 03/11] feat(testkit): add a ready subcommand to the test program `ready [--timeout SECS]` waits for the gateway the program will test and exits 2 if it is not ready in time. It shares its check with the `testkit-ready` binary, which now calls it instead of keeping its own copy. Signed-off-by: Evan Lezar --- .../testkit/testkit/src/bin/testkit-ready.rs | 29 +++--------- tests/testkit/testkit/src/lib.rs | 2 + tests/testkit/testkit/src/program/mod.rs | 47 +++++++++++++++++-- tests/testkit/testkit/src/ready.rs | 36 ++++++++++++++ tests/testkit/testkit/tests/program.rs | 33 ++++++++++++- 5 files changed, 119 insertions(+), 28 deletions(-) create mode 100644 tests/testkit/testkit/src/ready.rs diff --git a/tests/testkit/testkit/src/bin/testkit-ready.rs b/tests/testkit/testkit/src/bin/testkit-ready.rs index 4f41b3bed4..03db9d3547 100644 --- a/tests/testkit/testkit/src/bin/testkit-ready.rs +++ b/tests/testkit/testkit/src/bin/testkit-ready.rs @@ -6,13 +6,12 @@ //! Usage: `testkit-ready [TIMEOUT_SECS]` (default 60), with the gateway //! selected as for tests. Run it once per environment before the test //! binaries; exits non-zero with the last error if the gateway is not healthy -//! in time, or if the CLI does not reach it. +//! in time, or if the CLI does not reach it. A test program's `ready` +//! subcommand does the same. use std::process::ExitCode; use std::time::Duration; -use testkit::{Cli, Gateway}; - const DEFAULT_TIMEOUT_SECS: u64 = 60; const USAGE: &str = "usage: testkit-ready [TIMEOUT_SECS] (default 60)"; @@ -32,25 +31,9 @@ async fn main() -> ExitCode { } }, }; - let result = async { - let gateway = Gateway::from_env()?; - eprintln!("testkit-ready: {}", gateway.describe()); - let health = gateway.wait_ready(Duration::from_secs(timeout)).await?; - Cli::from_env(&gateway).check_connected().await?; - testkit::Result::Ok(health) - } - .await; - match result { - Ok(health) => { - eprintln!( - "testkit-ready: gateway {} is healthy and the CLI reaches it", - health.version - ); - ExitCode::SUCCESS - } - Err(error) => { - eprintln!("testkit-ready: {error}"); - ExitCode::FAILURE - } + if testkit::check_ready("testkit-ready", Duration::from_secs(timeout)).await { + ExitCode::SUCCESS + } else { + ExitCode::FAILURE } } diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 4f5148b1a5..dee03cac53 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -34,6 +34,7 @@ pub mod gateway; mod image; mod poll; pub mod program; +mod ready; mod registry; mod run; @@ -49,6 +50,7 @@ pub use image::{BASE_IMAGE_ENV, DEFAULT_BASE_IMAGE, base_image}; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until}; pub use program::main; +pub use ready::check as check_ready; pub use registry::Test; pub use run::{TestInfo, run}; pub use testkit_macros::test; diff --git a/tests/testkit/testkit/src/program/mod.rs b/tests/testkit/testkit/src/program/mod.rs index 0216a6ad11..fad56df442 100644 --- a/tests/testkit/testkit/src/program/mod.rs +++ b/tests/testkit/testkit/src/program/mod.rs @@ -12,8 +12,10 @@ //! } //! ``` //! -//! It has one subcommand so far: `list [FILTER]...` prints each test's name -//! and description. +//! It has two subcommands so far: +//! +//! - `list [FILTER]...` prints each test's name and description. +//! - `ready [--timeout SECS]` waits for the gateway, as `testkit-ready` does. //! //! Any other arguments are libtest's, so the program also works as a //! `harness = false` test target under `cargo test` and nextest: it answers @@ -24,6 +26,7 @@ use std::ffi::OsString; use std::path::PathBuf; use std::process::ExitCode; +use std::time::Duration; use clap::{Parser, Subcommand}; @@ -93,14 +96,33 @@ enum Command { /// Show tests whose name contains any of these (all if none). filters: Vec, }, + /// Wait until the gateway is healthy and the CLI reaches it. Exits 2 if + /// it is not ready in time. + Ready { + /// How long to wait, in seconds. + #[arg(long, default_value = "60", value_name = "SECS", value_parser = seconds)] + timeout: Duration, + }, +} + +/// A positive, finite number of seconds. +fn seconds(value: &str) -> Result { + let seconds: f64 = value + .parse() + .map_err(|_| format!("{value:?} is not a number"))?; + Duration::try_from_secs_f64(seconds) + .ok() + .filter(|duration| !duration.is_zero()) + .ok_or_else(|| format!("{value:?} is not a positive number of seconds")) } -fn command(_program: &str, tests: &'static [Test], command: Command) -> ExitCode { +fn command(program: &str, tests: &'static [Test], command: Command) -> ExitCode { match command { Command::List { filters } => { list(tests, filters); ExitCode::SUCCESS } + Command::Ready { timeout } => ready(program, timeout), } } @@ -131,6 +153,25 @@ fn describe(test: &Test) -> String { } } +fn ready(program: &str, timeout: Duration) -> ExitCode { + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + eprintln!("{program}: start the async runtime: {error}"); + return ExitCode::from(ERROR_EXIT); + } + }; + // A gateway that is not ready means the tests cannot run. + if runtime.block_on(crate::ready::check(program, timeout)) { + ExitCode::SUCCESS + } else { + ExitCode::from(ERROR_EXIT) + } +} + /// libtest's interface, for `cargo test` and nextest, and for `run`'s test /// processes. fn libtest(tests: &'static [Test], args: Vec) -> ExitCode { diff --git a/tests/testkit/testkit/src/ready.rs b/tests/testkit/testkit/src/ready.rs new file mode 100644 index 0000000000..db36f67659 --- /dev/null +++ b/tests/testkit/testkit/src/ready.rs @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Waiting until the gateway is ready for tests. + +use std::time::Duration; + +use crate::cli::Cli; +use crate::gateway::Gateway; + +/// Wait until the gateway selected as for tests is healthy and the CLI +/// reports it connected, logging to stderr as `program`. Returns whether it +/// is ready; if not, the last error has been logged. +pub async fn check(program: &str, timeout: Duration) -> bool { + let result = async { + let gateway = Gateway::from_env()?; + eprintln!("{program}: {}", gateway.describe()); + let health = gateway.wait_ready(timeout).await?; + Cli::from_env(&gateway).check_connected().await?; + crate::Result::Ok(health) + } + .await; + match result { + Ok(health) => { + eprintln!( + "{program}: gateway {} is healthy and the CLI reaches it", + health.version + ); + true + } + Err(error) => { + eprintln!("{program}: {error}"); + false + } + } +} diff --git a/tests/testkit/testkit/tests/program.rs b/tests/testkit/testkit/tests/program.rs index 16496c0b69..1f427db963 100644 --- a/tests/testkit/testkit/tests/program.rs +++ b/tests/testkit/testkit/tests/program.rs @@ -45,8 +45,9 @@ mod unix { use std::future::Future; use std::os::unix::process::CommandExt as _; use std::pin::Pin; + use std::path::Path; use std::process::{Command, Output, Stdio}; - use std::time::Duration; + use std::time::{Duration, Instant}; use testkit::{Test, TestInfo}; @@ -136,7 +137,7 @@ mod unix { }, ]; - pub static CHECKS: [Test; 3] = [ + pub static CHECKS: [Test; 4] = [ check("nextest_interface", || { Box::pin(async { nextest_interface() }) }), @@ -146,6 +147,9 @@ mod unix { check("a_program_without_tests_fails", || { Box::pin(async { a_program_without_tests_fails() }) }), + check("ready_fails_without_a_gateway", || { + Box::pin(async { ready_fails_without_a_gateway() }) + }), ]; fn program(fixture: &str, args: &[&str]) -> Command { @@ -253,4 +257,29 @@ mod unix { } } + /// Point the gateway lookup at a closed port, away from any registered + /// gateways. + fn without_gateway(mut command: Command, config: &Path) -> Command { + let closed = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let endpoint = format!("http://{}", closed.local_addr().expect("address")); + drop(closed); + command + .env("XDG_CONFIG_HOME", config) + .env("OPENSHELL_GATEWAY_ENDPOINT", endpoint) + .env_remove("OPENSHELL_GATEWAY") + .env("OPENSHELL_BIN", config.join("no-openshell")); + command + } + + fn ready_fails_without_a_gateway() { + let dir = tempfile::tempdir().expect("temp dir"); + let started = Instant::now(); + let ready = output(without_gateway( + program("registered", &["ready", "--timeout", "1"]), + dir.path(), + )); + assert_eq!(ready.status.code(), Some(2)); + assert!(stderr(&ready).contains("not ready after 1s")); + assert!(started.elapsed() < Duration::from_secs(10)); + } } From 9c65abf27daa2e69109308133e5f89654992e909 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 4 Nov 2026 14:39:51 +0100 Subject: [PATCH 04/11] feat(testkit): add a run subcommand to the test program `run [FILTER]... [--include-ignored] [--timeout SECS] [--grace SECS]` runs the selected tests one at a time. Each test is a new process, the program started as ` --exact --nocapture`, in its own process group, so nothing a test starts outlives it. A test that runs past its timeout (900s by default) gets SIGTERM, then SIGKILL after the grace period (10s by default). Interrupting `run` stops the current test and runs no more; a second interrupt skips the grace period. `run` exits 1 when a test failed and 2 when tests could not run or the run was interrupted. It is Unix only; on other platforms it says to use cargo test or nextest. Signed-off-by: Evan Lezar --- tests/testkit/testkit/src/program/mod.rs | 132 +++++++- tests/testkit/testkit/src/program/process.rs | 216 +++++++++++++ .../testkit/testkit/src/program/supervisor.rs | 305 ++++++++++++++++++ tests/testkit/testkit/tests/program.rs | 165 +++++++++- 4 files changed, 813 insertions(+), 5 deletions(-) create mode 100644 tests/testkit/testkit/src/program/process.rs create mode 100644 tests/testkit/testkit/src/program/supervisor.rs diff --git a/tests/testkit/testkit/src/program/mod.rs b/tests/testkit/testkit/src/program/mod.rs index fad56df442..3e6a6cfbbc 100644 --- a/tests/testkit/testkit/src/program/mod.rs +++ b/tests/testkit/testkit/src/program/mod.rs @@ -12,10 +12,12 @@ //! } //! ``` //! -//! It has two subcommands so far: +//! It has three subcommands: //! //! - `list [FILTER]...` prints each test's name and description. //! - `ready [--timeout SECS]` waits for the gateway, as `testkit-ready` does. +//! - `run [FILTER]...` runs tests one at a time, each in its own process with +//! a timeout. Unix only. //! //! Any other arguments are libtest's, so the program also works as a //! `harness = false` test target under `cargo test` and nextest: it answers @@ -23,6 +25,11 @@ //! In that mode tests run in the program's own process, one at a time unless //! `--test-threads` says otherwise. +#[cfg(unix)] +mod process; +#[cfg(unix)] +mod supervisor; + use std::ffi::OsString; use std::path::PathBuf; use std::process::ExitCode; @@ -32,6 +39,8 @@ use clap::{Parser, Subcommand}; use crate::registry::{RESERVED_NAMES, Selection, TESTS, Test, validate}; +/// Exit code when a test failed. +const FAILED_EXIT: u8 = 1; /// Exit code when tests could not run, including usage errors. const ERROR_EXIT: u8 = 2; /// Set by `run` for the test processes it starts. @@ -103,6 +112,22 @@ enum Command { #[arg(long, default_value = "60", value_name = "SECS", value_parser = seconds)] timeout: Duration, }, + /// Run tests one at a time, each in its own process. Exits 1 if a test + /// failed and 2 if tests could not run. + Run { + /// Run tests whose name contains any of these (all if none). + filters: Vec, + /// Run ignored tests too. + #[arg(long)] + include_ignored: bool, + /// Stop a test that runs longer than this, in seconds. Longer than + /// testkit's own step timeouts, so those report first. + #[arg(long, default_value = "900", value_name = "SECS", value_parser = seconds)] + timeout: Duration, + /// How long a stopped test has between SIGTERM and SIGKILL, in seconds. + #[arg(long, default_value = "10", value_name = "SECS", value_parser = seconds)] + grace: Duration, + }, } /// A positive, finite number of seconds. @@ -123,9 +148,61 @@ fn command(program: &str, tests: &'static [Test], command: Command) -> ExitCode ExitCode::SUCCESS } Command::Ready { timeout } => ready(program, timeout), + Command::Run { + filters, + include_ignored, + timeout, + grace, + } => run( + program, + tests, + Selection { + filters, + include_ignored, + }, + timeout, + grace, + ), } } +#[cfg(unix)] +fn run( + program: &str, + tests: &'static [Test], + selection: Selection, + timeout: Duration, + grace: Duration, +) -> ExitCode { + let exe = match std::env::current_exe() { + Ok(exe) => exe, + Err(error) => { + eprintln!("{program}: find this program's path: {error}"); + return ExitCode::from(ERROR_EXIT); + } + }; + let options = supervisor::Options { + selection, + timeout, + grace, + }; + supervisor::run(program, &exe, tests, &options) +} + +#[cfg(not(unix))] +fn run( + program: &str, + _tests: &'static [Test], + _selection: Selection, + _timeout: Duration, + _grace: Duration, +) -> ExitCode { + eprintln!( + "{program}: `run` is only supported on Unix; run the tests with cargo test or nextest" + ); + ExitCode::from(ERROR_EXIT) +} + /// One line per test: name and description, in aligned columns. /// An ignored test's description ends with `[ignored]` or its reason. fn list(tests: &[Test], filters: Vec) { @@ -230,10 +307,61 @@ fn program_name(args: &[OsString]) -> String { #[cfg(test)] mod tests { - use super::Cli; + use std::time::Duration; + + use clap::Parser; + + use super::{Cli, Command}; #[test] fn cli_is_consistent() { ::command().debug_assert(); } + + #[test] + fn run_parses_selection_and_limits() { + let cli = Cli::try_parse_from([ + "p", + "run", + "smoke", + "life", + "--include-ignored", + "--timeout", + "1.5", + ]) + .unwrap(); + let Command::Run { + filters, + include_ignored, + timeout, + grace, + } = cli.command + else { + panic!("not run"); + }; + assert_eq!(filters, ["smoke", "life"]); + assert!(include_ignored); + assert_eq!(timeout, Duration::from_millis(1500)); + assert_eq!(grace, Duration::from_secs(10)); + + let Command::Ready { timeout } = Cli::try_parse_from(["p", "ready"]).unwrap().command + else { + panic!("not ready"); + }; + assert_eq!(timeout, Duration::from_secs(60)); + } + + #[test] + fn rejects_bad_limits() { + for bad in ["0", "-1", "soon", "inf", "NaN"] { + assert!( + Cli::try_parse_from(["p", "run", "--timeout", bad]).is_err(), + "{bad}" + ); + assert!( + Cli::try_parse_from(["p", "ready", "--timeout", bad]).is_err(), + "{bad}" + ); + } + } } diff --git a/tests/testkit/testkit/src/program/process.rs b/tests/testkit/testkit/src/program/process.rs new file mode 100644 index 0000000000..1959452953 --- /dev/null +++ b/tests/testkit/testkit/src/program/process.rs @@ -0,0 +1,216 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One test process: started in its own process group, waited for with a +//! timeout, and stopped with `SIGTERM`, a grace period, then `SIGKILL`. +//! +//! The process group is killed whenever the test process ends, so a test's +//! helpers, such as CLI processes, cannot outlive it. + +use std::io; +use std::os::unix::process::CommandExt as _; +use std::process::{Child, Command, ExitStatus}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +const POLL_INTERVAL: Duration = Duration::from_millis(20); + +/// How the test process ended. +#[derive(Debug)] +pub enum Ended { + Exited(ExitStatus), + /// Stopped after the timeout. + TimedOut, + /// Stopped because the supervisor was asked to stop. + Interrupted, +} + +/// Start `command` in its own process group. +pub fn spawn(command: &mut Command) -> io::Result { + command.process_group(0).spawn() +} + +/// Wait for `child`, stopping it after `timeout` or once the supervisor has +/// been asked to stop (see [`install_stop_handlers`]). The child is reaped +/// and its group killed on every path, including errors. +pub fn wait(child: &mut Child, timeout: Duration, grace: Duration) -> io::Result { + let result = wait_or_stop(child, timeout, grace); + if result.is_err() { + let _ = reap(child); + } + result +} + +fn wait_or_stop(child: &mut Child, timeout: Duration, grace: Duration) -> io::Result { + let deadline = Instant::now() + timeout; + loop { + if has_exited(child)? { + return Ok(Ended::Exited(reap(child)?)); + } + if stop_requested() { + stop(child, grace)?; + return Ok(Ended::Interrupted); + } + if Instant::now() >= deadline { + stop(child, grace)?; + return Ok(Ended::TimedOut); + } + std::thread::sleep(POLL_INTERVAL); + } +} + +/// `SIGTERM` the group, wait up to `grace`, then `SIGKILL` it. A stop signal +/// during the grace period, or a second one in all, cuts it short. +fn stop(child: &mut Child, grace: Duration) -> io::Result { + signal_group(child, libc::SIGTERM); + let deadline = Instant::now() + grace; + let before = stop_requests(); + let cut_short = || { + let now = stop_requests(); + now > before || now >= 2 + }; + while Instant::now() < deadline && !cut_short() && !has_exited(child)? { + std::thread::sleep(POLL_INTERVAL); + } + reap(child) +} + +/// Kill whatever is left of the group, then collect the exit status. +fn reap(child: &mut Child) -> io::Result { + signal_group(child, libc::SIGKILL); + child.wait() +} + +static STOP_REQUESTS: AtomicUsize = AtomicUsize::new(0); + +fn stop_requests() -> usize { + STOP_REQUESTS.load(Ordering::SeqCst) +} + +/// Whether the supervisor has been asked to stop since it started. +pub fn stop_requested() -> bool { + stop_requests() > 0 +} + +fn pid(child: &Child) -> libc::pid_t { + libc::pid_t::try_from(child.id()).expect("pid fits in pid_t") +} + +/// Whether the child has exited, without reaping it. While it is unreaped +/// its PID, and so its process group ID, cannot be reused, so signalling the +/// group cannot reach an unrelated process. +#[cfg(target_os = "linux")] +fn has_exited(child: &mut Child) -> io::Result { + // SAFETY: `info` is a valid out-pointer, and `WNOWAIT` leaves the child + // for `Child::wait` to reap. + unsafe { + let mut info: libc::siginfo_t = std::mem::zeroed(); + let result = libc::waitid( + libc::P_PID, + child.id(), + &raw mut info, + libc::WEXITED | libc::WNOHANG | libc::WNOWAIT, + ); + if result == -1 { + return Err(io::Error::last_os_error()); + } + Ok(info.si_pid() != 0) + } +} + +/// Whether the child has exited, reaping it. `waitid` with `WNOWAIT` is not +/// reliable outside Linux (see golang/go#19314), so the group is killed just +/// after the leader is reaped. Its ID could in principle be reused in that +/// window, though a new process would have to get the same PID and lead a +/// group. +#[cfg(not(target_os = "linux"))] +fn has_exited(child: &mut Child) -> io::Result { + Ok(child.try_wait()?.is_some()) +} + +fn signal_group(child: &Child, signal: libc::c_int) { + // SAFETY: plain syscall to the child's own group (see `spawn`). ESRCH, an + // empty group, is fine. + unsafe { + libc::kill(-pid(child), signal); + } +} + +extern "C" fn on_stop(_signal: libc::c_int) { + STOP_REQUESTS.fetch_add(1, Ordering::SeqCst); +} + +/// Count `SIGINT`, `SIGTERM`, and `SIGHUP` as stop requests instead of dying. +/// Tests run in their own process group, so a Ctrl-C at the terminal reaches +/// only the supervisor, which must stop the test. +pub fn install_stop_handlers() -> io::Result<()> { + for signal in [libc::SIGINT, libc::SIGTERM, libc::SIGHUP] { + // SAFETY: the handler only touches an atomic, which is + // async-signal-safe. + unsafe { + let mut action: libc::sigaction = std::mem::zeroed(); + action.sa_sigaction = on_stop as *const () as libc::sighandler_t; + action.sa_flags = libc::SA_RESTART; + libc::sigemptyset(&raw mut action.sa_mask); + if libc::sigaction(signal, &raw const action, std::ptr::null_mut()) == -1 { + return Err(io::Error::last_os_error()); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::process::{Command, Stdio}; + use std::time::{Duration, Instant}; + + use super::{Ended, spawn, wait}; + + fn sh(script: &str) -> Command { + let mut command = Command::new("sh"); + command.args(["-c", script]).stdin(Stdio::null()); + command + } + + #[test] + fn exit_status_is_kept() { + let mut child = spawn(&mut sh("exit 3")).unwrap(); + let ended = wait(&mut child, Duration::from_secs(10), Duration::from_secs(1)).unwrap(); + assert!( + matches!(ended, Ended::Exited(status) if status.code() == Some(3)), + "{ended:?}" + ); + } + + #[test] + fn timeout_terminates_the_test() { + let started = Instant::now(); + let mut child = spawn(&mut sh("sleep 30")).unwrap(); + let ended = wait( + &mut child, + Duration::from_millis(200), + Duration::from_secs(10), + ) + .unwrap(); + assert!(matches!(ended, Ended::TimedOut), "{ended:?}"); + // `sleep` dies on SIGTERM, well inside the grace period. + assert!(started.elapsed() < Duration::from_secs(5)); + } + + #[test] + fn a_test_ignoring_sigterm_is_killed_after_the_grace_period() { + let started = Instant::now(); + let mut child = spawn(&mut sh("trap '' TERM; sleep 30")).unwrap(); + let ended = wait( + &mut child, + Duration::from_millis(200), + Duration::from_millis(500), + ) + .unwrap(); + assert!(matches!(ended, Ended::TimedOut), "{ended:?}"); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(700), "{elapsed:?}"); + assert!(elapsed < Duration::from_secs(5), "{elapsed:?}"); + } +} diff --git a/tests/testkit/testkit/src/program/supervisor.rs b/tests/testkit/testkit/src/program/supervisor.rs new file mode 100644 index 0000000000..21c3197e2c --- /dev/null +++ b/tests/testkit/testkit/src/program/supervisor.rs @@ -0,0 +1,305 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `run`: run the selected tests one at a time, each in a new process. +//! +//! The program starts itself once per test as ` --exact --nocapture`, +//! the same interface nextest uses, in a new process group. A test that runs +//! past its timeout gets `SIGTERM`, a grace period, then `SIGKILL`. Output is +//! shown as it comes. +//! +//! A test stopped this way does not run its cleanup: testkit tests do not +//! handle `SIGTERM` yet, so sandboxes named `test--*` can be left +//! behind. + +use std::fmt::Write as _; +use std::path::Path; +use std::process::{Command, ExitCode, ExitStatus, Stdio}; +use std::time::{Duration, Instant}; + +use super::process::{self, Ended}; +use super::{ERROR_EXIT, FAILED_EXIT, SUPERVISED_ENV}; +use crate::registry::{Disposition, Selection, Test}; + +/// libtest's exit code when a test failed. +const LIBTEST_FAILED: i32 = 101; + +pub struct Options { + pub selection: Selection, + pub timeout: Duration, + pub grace: Duration, +} + +/// One test's result. +pub struct TestRun<'a> { + pub test: &'a Test, + pub outcome: Outcome, + /// `None` for a test that did not run. + pub duration: Option, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Outcome { + Passed, + /// Ignored and not asked for. + Skipped, + Failed(String), + TimedOut(String), + /// The test could not be run. + Error(String), + /// Stopped because the run was interrupted. + Interrupted, +} + +impl Outcome { + fn label(&self) -> &'static str { + match self { + Self::Passed => "PASS", + Self::Skipped => "SKIP", + Self::Failed(_) => "FAIL", + Self::TimedOut(_) => "TIMEOUT", + Self::Error(_) | Self::Interrupted => "ERROR", + } + } + + fn message(&self) -> Option<&str> { + match self { + Self::Passed | Self::Skipped => None, + Self::Failed(message) | Self::TimedOut(message) | Self::Error(message) => Some(message), + Self::Interrupted => Some("interrupted"), + } + } +} + +pub fn run(program: &str, exe: &Path, tests: &[Test], options: &Options) -> ExitCode { + let selected = options.selection.apply(tests); + let to_run = selected + .iter() + .filter(|(_, disposition)| *disposition == Disposition::Run) + .count(); + if to_run == 0 { + eprintln!("{program}: no tests to run; `{program} list` shows the tests"); + return ExitCode::from(ERROR_EXIT); + } + if let Err(error) = process::install_stop_handlers() { + eprintln!("{program}: install signal handlers: {error}"); + return ExitCode::from(ERROR_EXIT); + } + let skipped = selected.len() - to_run; + eprintln!( + "{program}: running {to_run} test{} one at a time{}", + plural(to_run), + if skipped > 0 { + format!(", skipping {skipped} ignored") + } else { + String::new() + } + ); + + let started = Instant::now(); + let selected_count = selected.len(); + let mut runs = Vec::with_capacity(selected_count); + let mut position = 0; + for (test, disposition) in selected { + // A stop signal that arrives between tests still ends the run. + if process::stop_requested() { + break; + } + if disposition == Disposition::Skip { + runs.push(TestRun { + test, + outcome: Outcome::Skipped, + duration: None, + }); + continue; + } + position += 1; + let name = test.name(); + eprintln!("==> [{position}/{to_run}] {name}"); + let run = run_one(exe, test, options); + let elapsed = run.duration.unwrap_or_default(); + match run.outcome.message() { + None => eprintln!("<== {} {name} ({elapsed:.1?})", run.outcome.label()), + Some(message) => eprintln!( + "<== {} {name} ({elapsed:.1?}): {message}", + run.outcome.label() + ), + } + runs.push(run); + } + + let not_run = selected_count - runs.len(); + let status = summarize(program, &runs, not_run, started.elapsed()); + ExitCode::from(status) +} + +fn run_one<'a>(exe: &Path, test: &'a Test, options: &Options) -> TestRun<'a> { + let name = test.name(); + let mut command = Command::new(exe); + command + .args([name.as_str(), "--exact", "--nocapture"]) + .env(SUPERVISED_ENV, "1") + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + if test.ignored { + command.arg("--ignored"); + } + + let started = Instant::now(); + let mut child = match process::spawn(&mut command) { + Ok(child) => child, + Err(error) => { + return TestRun { + test, + outcome: Outcome::Error(format!("start {}: {error}", exe.display())), + duration: Some(started.elapsed()), + }; + } + }; + let ended = process::wait(&mut child, options.timeout, options.grace); + let duration = started.elapsed(); + let outcome = match ended { + Ok(ended) => outcome(ended, options.timeout), + Err(error) => Outcome::Error(format!("wait for the test process: {error}")), + }; + TestRun { + test, + outcome, + duration: Some(duration), + } +} + +fn outcome(ended: Ended, timeout: Duration) -> Outcome { + match ended { + Ended::Exited(status) => exited(status), + Ended::TimedOut => Outcome::TimedOut(format!( + "timed out after {}; stopped without cleanup", + seconds(timeout) + )), + Ended::Interrupted => Outcome::Interrupted, + } +} + +fn exited(status: ExitStatus) -> Outcome { + match status.code() { + Some(0) => Outcome::Passed, + Some(LIBTEST_FAILED) => Outcome::Failed("test failed".to_string()), + Some(code) if code == i32::from(ERROR_EXIT) => { + Outcome::Error("the test process could not run the test; see its output".to_string()) + } + Some(code) => Outcome::Failed(format!("test process exited with code {code}")), + None => Outcome::Failed(format!("test process ended by {status}")), + } +} + +/// Print the totals and the tests that did not pass; return the exit code: +/// 1 if a test failed, 2 if one could not run or the run was interrupted +/// before `not_run` tests. +fn summarize(program: &str, runs: &[TestRun<'_>], not_run: usize, elapsed: Duration) -> u8 { + let count = + |predicate: fn(&Outcome) -> bool| runs.iter().filter(|run| predicate(&run.outcome)).count(); + let passed = count(|outcome| matches!(outcome, Outcome::Passed)); + let failed = count(|outcome| matches!(outcome, Outcome::Failed(_) | Outcome::TimedOut(_))); + let errors = count(|outcome| matches!(outcome, Outcome::Error(_) | Outcome::Interrupted)); + let skipped = count(|outcome| matches!(outcome, Outcome::Skipped)); + let mut summary = format!("{program}: {passed} passed, {failed} failed"); + if errors > 0 { + let _ = write!(summary, ", {errors} could not run"); + } + if skipped > 0 { + let _ = write!(summary, ", {skipped} skipped"); + } + let _ = write!(summary, " in {elapsed:.1?}"); + eprintln!("{summary}"); + for run in runs { + if let Some(message) = run.outcome.message() { + eprintln!(" {} {}: {message}", run.outcome.label(), run.test.name()); + } + } + if not_run > 0 { + eprintln!( + "{program}: interrupted; {not_run} later test{} did not run", + plural(not_run) + ); + } + if errors > 0 || not_run > 0 { + ERROR_EXIT + } else if failed > 0 { + FAILED_EXIT + } else { + 0 + } +} + +fn seconds(duration: Duration) -> String { + format!("{}s", duration.as_secs_f64()) +} + +fn plural(count: usize) -> &'static str { + if count == 1 { "" } else { "s" } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::{Outcome, TestRun, summarize}; + use crate::registry::Test; + use crate::registry::tests::test; + + const TEST: Test = test("suite::m", "t", false); + + fn runs(outcomes: Vec) -> Vec> { + outcomes + .into_iter() + .map(|outcome| TestRun { + test: &TEST, + outcome, + duration: None, + }) + .collect() + } + + #[test] + fn exit_code_separates_failures_from_runs_that_could_not_complete() { + let code = |outcomes| summarize("p", &runs(outcomes), 0, Duration::ZERO); + assert_eq!(code(vec![Outcome::Passed, Outcome::Skipped]), 0); + assert_eq!(code(vec![Outcome::Passed, Outcome::Failed("f".into())]), 1); + assert_eq!(code(vec![Outcome::TimedOut("t".into())]), 1); + assert_eq!( + code(vec![ + Outcome::Failed("f".into()), + Outcome::Error("e".into()) + ]), + 2 + ); + assert_eq!(code(vec![Outcome::Interrupted]), 2); + // Interrupted between tests: every test that ran passed. + assert_eq!( + summarize("p", &runs(vec![Outcome::Passed]), 1, Duration::ZERO), + 2 + ); + } + + #[test] + fn exit_statuses_map_to_outcomes() { + use std::os::unix::process::ExitStatusExt; + + use super::exited; + + let status = |code: i32| std::process::ExitStatus::from_raw(code << 8); + assert_eq!(exited(status(0)), Outcome::Passed); + assert_eq!(exited(status(101)), Outcome::Failed("test failed".into())); + assert!(matches!(exited(status(2)), Outcome::Error(_))); + assert_eq!( + exited(status(7)), + Outcome::Failed("test process exited with code 7".into()) + ); + let killed = exited(std::process::ExitStatus::from_raw(libc::SIGSEGV)); + assert!( + matches!(&killed, Outcome::Failed(message) if message.contains("signal")), + "{killed:?}" + ); + } +} diff --git a/tests/testkit/testkit/tests/program.rs b/tests/testkit/testkit/tests/program.rs index 1f427db963..d7132e38b9 100644 --- a/tests/testkit/testkit/tests/program.rs +++ b/tests/testkit/testkit/tests/program.rs @@ -44,9 +44,9 @@ async fn registered_ignored(_cx: &mut testkit::Context) -> testkit::Result { mod unix { use std::future::Future; use std::os::unix::process::CommandExt as _; - use std::pin::Pin; use std::path::Path; - use std::process::{Command, Output, Stdio}; + use std::pin::Pin; + use std::process::{Child, Command, Output, Stdio}; use std::time::{Duration, Instant}; use testkit::{Test, TestInfo}; @@ -137,7 +137,7 @@ mod unix { }, ]; - pub static CHECKS: [Test; 4] = [ + pub static CHECKS: [Test; 11] = [ check("nextest_interface", || { Box::pin(async { nextest_interface() }) }), @@ -150,6 +150,25 @@ mod unix { check("ready_fails_without_a_gateway", || { Box::pin(async { ready_fails_without_a_gateway() }) }), + check("run_reports_every_outcome", || { + Box::pin(async { run_reports_every_outcome() }) + }), + check("run_exit_codes", || Box::pin(async { run_exit_codes() })), + check("timeout_kills_the_process_group", || { + Box::pin(async { timeout_kills_the_process_group() }) + }), + check("sigterm_is_followed_by_sigkill", || { + Box::pin(async { sigterm_is_followed_by_sigkill() }) + }), + check("interrupt_stops_the_test_and_the_run", || { + Box::pin(async { interrupt_stops_the_test_and_the_run() }) + }), + check("a_passing_test_leaves_nothing_behind", || { + Box::pin(async { a_passing_test_leaves_nothing_behind() }) + }), + check("registered_tests_run_through_testkit", || { + Box::pin(async { registered_tests_run_through_testkit() }) + }), ]; fn program(fixture: &str, args: &[&str]) -> Command { @@ -282,4 +301,144 @@ mod unix { assert!(stderr(&ready).contains("not ready after 1s")); assert!(started.elapsed() < Duration::from_secs(10)); } + + fn is_alive(pid: libc::pid_t) -> bool { + // SAFETY: signal 0 only checks the process exists. + unsafe { libc::kill(pid, 0) == 0 } + } + + /// Wait for a killed helper to go: once its parent is gone, init reaps it. + fn assert_gone(pid: libc::pid_t) { + let deadline = Instant::now() + Duration::from_secs(10); + while is_alive(pid) { + assert!(Instant::now() < deadline, "helper {pid} outlived its test"); + std::thread::sleep(Duration::from_millis(50)); + } + } + + fn read_pid(path: &Path) -> libc::pid_t { + let deadline = Instant::now() + Duration::from_secs(30); + loop { + if let Ok(pid) = std::fs::read_to_string(path) + && let Ok(pid) = pid.trim().parse() + { + return pid; + } + assert!(Instant::now() < deadline, "the hanging test never started"); + std::thread::sleep(Duration::from_millis(20)); + } + } + + fn run_reports_every_outcome() { + let run = output(program("fixtures", &["run", "passes", "fails", "ignored"])); + assert_eq!(run.status.code(), Some(1)); + // Test output is passed through. + assert!(stdout(&run).contains("passes: stdout")); + assert!(stderr(&run).contains("passes: stderr")); + assert!(stderr(&run).contains("1 passed, 1 failed, 1 skipped")); + assert!(stderr(&run).contains("FAIL fixture::fails: test failed")); + } + + fn run_exit_codes() { + let pass = output(program("fixtures", &["run", "passes"])); + assert_eq!(pass.status.code(), Some(0)); + let ignored = output(program( + "fixtures", + &["run", "ignored", "--include-ignored"], + )); + assert_eq!(ignored.status.code(), Some(0)); + assert!(stderr(&ignored).contains("1 passed, 0 failed")); + // Selecting nothing to run is not a pass. + for args in [&["run", "nothing-matches"][..], &["run", "ignored"]] { + let none = output(program("fixtures", args)); + assert_eq!(none.status.code(), Some(2), "{args:?}"); + assert!(stderr(&none).contains("no tests to run")); + } + let usage = output(program("fixtures", &["run", "--timeout", "0"])); + assert_eq!(usage.status.code(), Some(2)); + } + + fn timeout_kills_the_process_group() { + let dir = tempfile::tempdir().expect("temp dir"); + let pid_file = dir.path().join("pid"); + let mut command = program( + "fixtures", + &["run", "hangs", "--timeout", "1", "--grace", "5"], + ); + command.env(PID_FILE_ENV, &pid_file); + let started = Instant::now(); + let run = output(command); + assert_eq!(run.status.code(), Some(1)); + // `SIGTERM` stops the test well inside the grace period. + assert!( + started.elapsed() < Duration::from_secs(5), + "{:?}", + started.elapsed() + ); + assert!(stderr(&run).contains("TIMEOUT fixture::hangs: timed out after 1s")); + assert_gone(read_pid(&pid_file)); + } + + fn sigterm_is_followed_by_sigkill() { + let started = Instant::now(); + let run = output(program( + "fixtures", + &[ + "run", + "ignores_sigterm", + "--timeout", + "0.2", + "--grace", + "0.5", + ], + )); + assert_eq!(run.status.code(), Some(1)); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(700), "{elapsed:?}"); + assert!(elapsed < Duration::from_secs(10), "{elapsed:?}"); + assert!(stderr(&run).contains("TIMEOUT fixture::ignores_sigterm")); + } + + fn interrupt_stops_the_test_and_the_run() { + let dir = tempfile::tempdir().expect("temp dir"); + let pid_file = dir.path().join("pid"); + let mut command = program("fixtures", &["run", "hangs", "passes", "--grace", "5"]); + command + .env(PID_FILE_ENV, &pid_file) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let child: Child = command.spawn().expect("start the run"); + let helper = read_pid(&pid_file); + // SAFETY: plain syscall to our own child, not yet reaped. + unsafe { + libc::kill( + libc::pid_t::try_from(child.id()).expect("pid"), + libc::SIGTERM, + ); + } + let run = child.wait_with_output().expect("wait for the run"); + eprintln!("{}", String::from_utf8_lossy(&run.stderr)); + assert_eq!(run.status.code(), Some(2)); + assert!(stderr(&run).contains("ERROR fixture::hangs: interrupted")); + assert_gone(helper); + } + + fn a_passing_test_leaves_nothing_behind() { + let dir = tempfile::tempdir().expect("temp dir"); + let pid_file = dir.path().join("pid"); + let mut command = program("fixtures", &["run", "leaves_a_helper"]); + command.env(PID_FILE_ENV, &pid_file); + let run = output(command); + assert_eq!(run.status.code(), Some(0)); + assert_gone(read_pid(&pid_file)); + } + + fn registered_tests_run_through_testkit() { + let dir = tempfile::tempdir().expect("temp dir"); + let run = output(without_gateway(program("registered", &["run"]), dir.path())); + assert_eq!(run.status.code(), Some(1)); + // `testkit::run` logs the description and the gateway. + assert!(stderr(&run).contains("[registered_needs_a_gateway] Needs a gateway.")); + assert!(stderr(&run).contains("1 failed, 1 skipped")); + } } From 4b51727a76520e60a47675d57913b5384470e971 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 4 Nov 2026 14:40:23 +0100 Subject: [PATCH 05/11] feat(testkit): write a JUnit report from the test program's run `run --junit PATH` writes a JUnit report with one test suite per family and one test case per test, including skipped, failed and timed-out tests. Each test's output is shown as it comes and the last 4 MiB of each stream is kept for the report. The supervisor waits for a test's output to close before it moves on, so a signal that arrives while it waits still ends the run. Signed-off-by: Evan Lezar --- tests/testkit/testkit/src/program/capture.rs | 109 +++++++++ tests/testkit/testkit/src/program/junit.rs | 128 +++++++++++ tests/testkit/testkit/src/program/mod.rs | 18 +- .../testkit/testkit/src/program/supervisor.rs | 41 +++- tests/testkit/testkit/tests/program.rs | 215 ++++++++++++------ 5 files changed, 438 insertions(+), 73 deletions(-) create mode 100644 tests/testkit/testkit/src/program/capture.rs create mode 100644 tests/testkit/testkit/src/program/junit.rs diff --git a/tests/testkit/testkit/src/program/capture.rs b/tests/testkit/testkit/src/program/capture.rs new file mode 100644 index 0000000000..329c22ac68 --- /dev/null +++ b/tests/testkit/testkit/src/program/capture.rs @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Copying a test's output to the console while keeping it for the report. + +use std::collections::VecDeque; +use std::io::{self, Read, Write}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +/// How much of each stream is kept for the report. Output beyond this is +/// still shown; the report keeps the end, where failures are. +const KEPT_BYTES: usize = 4 << 20; + +/// How long to wait for a stream to close after the test's process group is +/// gone. A process that left the group can hold it open. +const CLOSE_TIMEOUT: Duration = Duration::from_secs(5); + +/// One stream being copied. +pub struct Capture { + kept: Arc>, + thread: JoinHandle<()>, +} + +#[derive(Default)] +struct Kept { + bytes: VecDeque, + dropped: usize, +} + +impl Capture { + /// Copy `stream` to `console` on a thread, keeping a copy. + pub fn start( + mut stream: impl Read + Send + 'static, + mut console: impl Write + Send + 'static, + ) -> Self { + let kept = Arc::new(Mutex::new(Kept::default())); + let thread = std::thread::spawn({ + let kept = Arc::clone(&kept); + move || { + let mut buffer = [0; 8192]; + loop { + let read = match stream.read(&mut buffer) { + Ok(0) => return, + Ok(read) => read, + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(_) => return, + }; + let chunk = &buffer[..read]; + let _ = console.write_all(chunk).and_then(|()| console.flush()); + kept.lock() + .unwrap_or_else(PoisonError::into_inner) + .push(chunk); + } + } + }); + Self { kept, thread } + } + + /// The output kept, once the stream closes or after a short wait. + pub fn finish(self) -> Vec { + let deadline = Instant::now() + CLOSE_TIMEOUT; + while !self.thread.is_finished() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + if self.thread.is_finished() { + let _ = self.thread.join(); + } + let kept = self.kept.lock().unwrap_or_else(PoisonError::into_inner); + let mut bytes = Vec::with_capacity(kept.bytes.len() + 64); + if kept.dropped > 0 { + bytes.extend(format!("[{} earlier bytes not kept]\n", kept.dropped).as_bytes()); + } + bytes.extend(&kept.bytes); + bytes + } +} + +impl Kept { + fn push(&mut self, chunk: &[u8]) { + self.bytes.extend(chunk); + let excess = self.bytes.len().saturating_sub(KEPT_BYTES); + self.bytes.drain(..excess); + self.dropped += excess; + } +} + +#[cfg(test)] +mod tests { + use super::{Capture, KEPT_BYTES, Kept}; + + #[test] + fn copies_and_keeps_the_stream() { + let capture = Capture::start(&b"one\ntwo\n"[..], std::io::sink()); + assert_eq!(capture.finish(), b"one\ntwo\n"); + } + + #[test] + fn keeps_the_end_of_long_output() { + let mut kept = Kept::default(); + kept.push(&vec![b'a'; KEPT_BYTES]); + kept.push(b"end"); + assert_eq!(kept.bytes.len(), KEPT_BYTES); + let tail: Vec = kept.bytes.iter().rev().take(4).rev().copied().collect(); + assert_eq!(tail, b"aend"); + assert_eq!(kept.dropped, 3); + } +} diff --git a/tests/testkit/testkit/src/program/junit.rs b/tests/testkit/testkit/src/program/junit.rs new file mode 100644 index 0000000000..0a1e66d648 --- /dev/null +++ b/tests/testkit/testkit/src/program/junit.rs @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The `JUnit` report of a run. +//! +//! One `` per family. Each `` is named by the test name, +//! with the module path as its class name and the description and family as +//! properties. + +use std::collections::BTreeMap; +use std::path::Path; + +use quick_junit::{NonSuccessKind, Property, Report, TestCase, TestCaseStatus, TestSuite}; + +use super::supervisor::{Outcome, TestRun}; + +pub fn report(program: &str, runs: &[TestRun<'_>]) -> Report { + let mut suites: BTreeMap<&str, TestSuite> = BTreeMap::new(); + for run in runs { + let family = run.test.family(); + suites + .entry(family) + .or_insert_with(|| TestSuite::new(family)) + .add_test_case(test_case(run)); + } + let mut report = Report::new(program); + report.add_test_suites(suites.into_values()); + report +} + +fn test_case(run: &TestRun<'_>) -> TestCase { + let status = match &run.outcome { + Outcome::Passed => TestCaseStatus::success(), + Outcome::Skipped => { + let mut status = TestCaseStatus::skipped(); + status.set_message(match run.test.ignore_reason { + Some(reason) => format!("ignored: {reason}"), + None => "ignored".to_string(), + }); + status + } + Outcome::Failed(message) => non_success(NonSuccessKind::Failure, "failure", message), + Outcome::TimedOut(message) => non_success(NonSuccessKind::Failure, "timeout", message), + Outcome::Error(message) => non_success(NonSuccessKind::Error, "error", message), + Outcome::Interrupted => non_success(NonSuccessKind::Error, "interrupted", "interrupted"), + }; + let mut case = TestCase::new(run.test.name(), status); + case.set_classname(run.test.module_path) + .add_property(Property::new("family", run.test.family())) + .add_property(Property::new("description", run.test.info.description)); + if let Some(duration) = run.duration { + case.set_time(duration); + } + if !run.stdout.is_empty() { + case.set_system_out_lossy(&run.stdout); + } + if !run.stderr.is_empty() { + case.set_system_err_lossy(&run.stderr); + } + case +} + +fn non_success(kind: NonSuccessKind, ty: &str, message: &str) -> TestCaseStatus { + let mut status = TestCaseStatus::non_success(kind); + status.set_type(ty).set_message(message); + status +} + +pub fn write(path: &Path, report: &Report) -> std::io::Result<()> { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent)?; + } + let xml = report.to_string().map_err(std::io::Error::other)?; + std::fs::write(path, xml) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::report; + use crate::program::supervisor::{Outcome, TestRun}; + use crate::registry::Test; + use crate::registry::tests::test; + + const PASSED: Test = test("sandbox::smoke", "create_exec_delete", false); + const FAILED: Test = test("sandbox::lifecycle", "stop_start", false); + const SKIPPED: Test = Test { + ignore_reason: Some("https://github.com/NVIDIA/OpenShell/issues/1"), + ..test("sandbox::lifecycle", "later", true) + }; + + #[test] + fn reports_every_outcome() { + let run = |test, outcome, duration| TestRun { + test, + outcome, + duration, + stdout: b"out\n".to_vec(), + stderr: Vec::new(), + }; + let runs = [ + run(&PASSED, Outcome::Passed, Some(Duration::from_millis(1500))), + run( + &FAILED, + Outcome::TimedOut("timed out after 1s".into()), + Some(Duration::from_secs(1)), + ), + run(&SKIPPED, Outcome::Skipped, None), + ]; + let xml = report("openshell-conformance", &runs).to_string().unwrap(); + for expected in [ + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + "out\n", + ] { + assert!(xml.contains(expected), "missing {expected} in\n{xml}"); + } + } +} diff --git a/tests/testkit/testkit/src/program/mod.rs b/tests/testkit/testkit/src/program/mod.rs index 3e6a6cfbbc..c7ca98f3ba 100644 --- a/tests/testkit/testkit/src/program/mod.rs +++ b/tests/testkit/testkit/src/program/mod.rs @@ -17,7 +17,7 @@ //! - `list [FILTER]...` prints each test's name and description. //! - `ready [--timeout SECS]` waits for the gateway, as `testkit-ready` does. //! - `run [FILTER]...` runs tests one at a time, each in its own process with -//! a timeout. Unix only. +//! a timeout, and can write a `JUnit` report. Unix only. //! //! Any other arguments are libtest's, so the program also works as a //! `harness = false` test target under `cargo test` and nextest: it answers @@ -25,6 +25,10 @@ //! In that mode tests run in the program's own process, one at a time unless //! `--test-threads` says otherwise. +#[cfg(unix)] +mod capture; +#[cfg(unix)] +mod junit; #[cfg(unix)] mod process; #[cfg(unix)] @@ -127,6 +131,9 @@ enum Command { /// How long a stopped test has between SIGTERM and SIGKILL, in seconds. #[arg(long, default_value = "10", value_name = "SECS", value_parser = seconds)] grace: Duration, + /// Write a JUnit report here. + #[arg(long, value_name = "PATH")] + junit: Option, }, } @@ -153,6 +160,7 @@ fn command(program: &str, tests: &'static [Test], command: Command) -> ExitCode include_ignored, timeout, grace, + junit, } => run( program, tests, @@ -162,6 +170,7 @@ fn command(program: &str, tests: &'static [Test], command: Command) -> ExitCode }, timeout, grace, + junit, ), } } @@ -173,6 +182,7 @@ fn run( selection: Selection, timeout: Duration, grace: Duration, + junit: Option, ) -> ExitCode { let exe = match std::env::current_exe() { Ok(exe) => exe, @@ -185,6 +195,7 @@ fn run( selection, timeout, grace, + junit, }; supervisor::run(program, &exe, tests, &options) } @@ -196,6 +207,7 @@ fn run( _selection: Selection, _timeout: Duration, _grace: Duration, + _junit: Option, ) -> ExitCode { eprintln!( "{program}: `run` is only supported on Unix; run the tests with cargo test or nextest" @@ -328,6 +340,8 @@ mod tests { "--include-ignored", "--timeout", "1.5", + "--junit", + "out/junit.xml", ]) .unwrap(); let Command::Run { @@ -335,6 +349,7 @@ mod tests { include_ignored, timeout, grace, + junit, } = cli.command else { panic!("not run"); @@ -343,6 +358,7 @@ mod tests { assert!(include_ignored); assert_eq!(timeout, Duration::from_millis(1500)); assert_eq!(grace, Duration::from_secs(10)); + assert_eq!(junit.unwrap().to_str(), Some("out/junit.xml")); let Command::Ready { timeout } = Cli::try_parse_from(["p", "ready"]).unwrap().command else { diff --git a/tests/testkit/testkit/src/program/supervisor.rs b/tests/testkit/testkit/src/program/supervisor.rs index 21c3197e2c..6dced12ae1 100644 --- a/tests/testkit/testkit/src/program/supervisor.rs +++ b/tests/testkit/testkit/src/program/supervisor.rs @@ -6,19 +6,20 @@ //! The program starts itself once per test as ` --exact --nocapture`, //! the same interface nextest uses, in a new process group. A test that runs //! past its timeout gets `SIGTERM`, a grace period, then `SIGKILL`. Output is -//! shown as it comes. +//! shown as it comes and kept for the `JUnit` report. //! //! A test stopped this way does not run its cleanup: testkit tests do not //! handle `SIGTERM` yet, so sandboxes named `test--*` can be left //! behind. use std::fmt::Write as _; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::{Command, ExitCode, ExitStatus, Stdio}; use std::time::{Duration, Instant}; +use super::capture::Capture; use super::process::{self, Ended}; -use super::{ERROR_EXIT, FAILED_EXIT, SUPERVISED_ENV}; +use super::{ERROR_EXIT, FAILED_EXIT, SUPERVISED_ENV, junit}; use crate::registry::{Disposition, Selection, Test}; /// libtest's exit code when a test failed. @@ -28,6 +29,7 @@ pub struct Options { pub selection: Selection, pub timeout: Duration, pub grace: Duration, + pub junit: Option, } /// One test's result. @@ -36,6 +38,8 @@ pub struct TestRun<'a> { pub outcome: Outcome, /// `None` for a test that did not run. pub duration: Option, + pub stdout: Vec, + pub stderr: Vec, } #[derive(Debug, PartialEq, Eq)] @@ -110,6 +114,8 @@ pub fn run(program: &str, exe: &Path, tests: &[Test], options: &Options) -> Exit test, outcome: Outcome::Skipped, duration: None, + stdout: Vec::new(), + stderr: Vec::new(), }); continue; } @@ -129,7 +135,16 @@ pub fn run(program: &str, exe: &Path, tests: &[Test], options: &Options) -> Exit } let not_run = selected_count - runs.len(); - let status = summarize(program, &runs, not_run, started.elapsed()); + let mut status = summarize(program, &runs, not_run, started.elapsed()); + if let Some(path) = &options.junit { + match junit::write(path, &junit::report(program, &runs)) { + Ok(()) => eprintln!("{program}: wrote {}", path.display()), + Err(error) => { + eprintln!("{program}: write {}: {error}", path.display()); + status = ERROR_EXIT; + } + } + } ExitCode::from(status) } @@ -140,8 +155,8 @@ fn run_one<'a>(exe: &Path, test: &'a Test, options: &Options) -> TestRun<'a> { .args([name.as_str(), "--exact", "--nocapture"]) .env(SUPERVISED_ENV, "1") .stdin(Stdio::null()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()); + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); if test.ignored { command.arg("--ignored"); } @@ -154,9 +169,19 @@ fn run_one<'a>(exe: &Path, test: &'a Test, options: &Options) -> TestRun<'a> { test, outcome: Outcome::Error(format!("start {}: {error}", exe.display())), duration: Some(started.elapsed()), + stdout: Vec::new(), + stderr: Vec::new(), }; } }; + let stdout = child + .stdout + .take() + .map(|stream| Capture::start(stream, std::io::stdout())); + let stderr = child + .stderr + .take() + .map(|stream| Capture::start(stream, std::io::stderr())); let ended = process::wait(&mut child, options.timeout, options.grace); let duration = started.elapsed(); let outcome = match ended { @@ -167,6 +192,8 @@ fn run_one<'a>(exe: &Path, test: &'a Test, options: &Options) -> TestRun<'a> { test, outcome, duration: Some(duration), + stdout: stdout.map(Capture::finish).unwrap_or_default(), + stderr: stderr.map(Capture::finish).unwrap_or_default(), } } @@ -257,6 +284,8 @@ mod tests { test: &TEST, outcome, duration: None, + stdout: Vec::new(), + stderr: Vec::new(), }) .collect() } diff --git a/tests/testkit/testkit/tests/program.rs b/tests/testkit/testkit/tests/program.rs index d7132e38b9..c414e8dd27 100644 --- a/tests/testkit/testkit/tests/program.rs +++ b/tests/testkit/testkit/tests/program.rs @@ -44,7 +44,7 @@ async fn registered_ignored(_cx: &mut testkit::Context) -> testkit::Result { mod unix { use std::future::Future; use std::os::unix::process::CommandExt as _; - use std::path::Path; + use std::path::{Path, PathBuf}; use std::pin::Pin; use std::process::{Child, Command, Output, Stdio}; use std::time::{Duration, Instant}; @@ -137,23 +137,20 @@ mod unix { }, ]; - pub static CHECKS: [Test; 11] = [ + pub static CHECKS: [Test; 12] = [ check("nextest_interface", || { Box::pin(async { nextest_interface() }) }), check("list_shows_name_and_description", || { Box::pin(async { list_shows_name_and_description() }) }), - check("a_program_without_tests_fails", || { - Box::pin(async { a_program_without_tests_fails() }) - }), - check("ready_fails_without_a_gateway", || { - Box::pin(async { ready_fails_without_a_gateway() }) - }), check("run_reports_every_outcome", || { Box::pin(async { run_reports_every_outcome() }) }), check("run_exit_codes", || Box::pin(async { run_exit_codes() })), + check("a_program_without_tests_fails", || { + Box::pin(async { a_program_without_tests_fails() }) + }), check("timeout_kills_the_process_group", || { Box::pin(async { timeout_kills_the_process_group() }) }), @@ -163,12 +160,18 @@ mod unix { check("interrupt_stops_the_test_and_the_run", || { Box::pin(async { interrupt_stops_the_test_and_the_run() }) }), + check("interrupt_between_tests_ends_the_run", || { + Box::pin(async { interrupt_between_tests_ends_the_run() }) + }), check("a_passing_test_leaves_nothing_behind", || { Box::pin(async { a_passing_test_leaves_nothing_behind() }) }), check("registered_tests_run_through_testkit", || { Box::pin(async { registered_tests_run_through_testkit() }) }), + check("ready_fails_without_a_gateway", || { + Box::pin(async { ready_fails_without_a_gateway() }) + }), ]; fn program(fixture: &str, args: &[&str]) -> Command { @@ -200,6 +203,11 @@ mod unix { std::str::from_utf8(&output.stderr).expect("utf-8") } + fn junit(dir: &Path) -> (PathBuf, String) { + let path = dir.join("results").join("junit.xml"); + (path.clone(), path.to_str().expect("utf-8 path").to_string()) + } + /// nextest lists with `--list --format terse [--ignored]` and runs each /// test with ` --exact --nocapture`. fn nextest_interface() { @@ -258,6 +266,55 @@ mod unix { assert_eq!(stdout(&filtered), "fixture::passes A fixture.\n"); } + fn run_reports_every_outcome() { + let dir = tempfile::tempdir().expect("temp dir"); + let (path, junit_arg) = junit(dir.path()); + let run = output(program( + "fixtures", + &["run", "passes", "fails", "ignored", "--junit", &junit_arg], + )); + assert_eq!(run.status.code(), Some(1)); + // Test output is passed through. + assert!(stdout(&run).contains("passes: stdout")); + assert!(stderr(&run).contains("passes: stderr")); + assert!(stderr(&run).contains("1 passed, 1 failed, 1 skipped")); + assert!(stderr(&run).contains("FAIL fixture::fails: test failed")); + + let xml = std::fs::read_to_string(path).expect("junit report"); + for expected in [ + r#""#, + r#""#, + r#""#, + "", + "passes: stdout", + "fixture failure", + ] { + assert!(xml.contains(expected), "missing {expected} in\n{xml}"); + } + } + + fn run_exit_codes() { + let pass = output(program("fixtures", &["run", "passes"])); + assert_eq!(pass.status.code(), Some(0)); + let ignored = output(program( + "fixtures", + &["run", "ignored", "--include-ignored"], + )); + assert_eq!(ignored.status.code(), Some(0)); + assert!(stderr(&ignored).contains("1 passed, 0 failed")); + // Selecting nothing to run is not a pass. + for args in [&["run", "nothing-matches"][..], &["run", "ignored"]] { + let none = output(program("fixtures", args)); + assert_eq!(none.status.code(), Some(2), "{args:?}"); + assert!(stderr(&none).contains("no tests to run")); + } + let usage = output(program("fixtures", &["run", "--timeout", "0"])); + assert_eq!(usage.status.code(), Some(2)); + } + /// A program whose suites were not linked fails rather than passing with /// no tests, under every interface. fn a_program_without_tests_fails() { @@ -276,32 +333,6 @@ mod unix { } } - /// Point the gateway lookup at a closed port, away from any registered - /// gateways. - fn without_gateway(mut command: Command, config: &Path) -> Command { - let closed = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); - let endpoint = format!("http://{}", closed.local_addr().expect("address")); - drop(closed); - command - .env("XDG_CONFIG_HOME", config) - .env("OPENSHELL_GATEWAY_ENDPOINT", endpoint) - .env_remove("OPENSHELL_GATEWAY") - .env("OPENSHELL_BIN", config.join("no-openshell")); - command - } - - fn ready_fails_without_a_gateway() { - let dir = tempfile::tempdir().expect("temp dir"); - let started = Instant::now(); - let ready = output(without_gateway( - program("registered", &["ready", "--timeout", "1"]), - dir.path(), - )); - assert_eq!(ready.status.code(), Some(2)); - assert!(stderr(&ready).contains("not ready after 1s")); - assert!(started.elapsed() < Duration::from_secs(10)); - } - fn is_alive(pid: libc::pid_t) -> bool { // SAFETY: signal 0 only checks the process exists. unsafe { libc::kill(pid, 0) == 0 } @@ -329,41 +360,22 @@ mod unix { } } - fn run_reports_every_outcome() { - let run = output(program("fixtures", &["run", "passes", "fails", "ignored"])); - assert_eq!(run.status.code(), Some(1)); - // Test output is passed through. - assert!(stdout(&run).contains("passes: stdout")); - assert!(stderr(&run).contains("passes: stderr")); - assert!(stderr(&run).contains("1 passed, 1 failed, 1 skipped")); - assert!(stderr(&run).contains("FAIL fixture::fails: test failed")); - } - - fn run_exit_codes() { - let pass = output(program("fixtures", &["run", "passes"])); - assert_eq!(pass.status.code(), Some(0)); - let ignored = output(program( - "fixtures", - &["run", "ignored", "--include-ignored"], - )); - assert_eq!(ignored.status.code(), Some(0)); - assert!(stderr(&ignored).contains("1 passed, 0 failed")); - // Selecting nothing to run is not a pass. - for args in [&["run", "nothing-matches"][..], &["run", "ignored"]] { - let none = output(program("fixtures", args)); - assert_eq!(none.status.code(), Some(2), "{args:?}"); - assert!(stderr(&none).contains("no tests to run")); - } - let usage = output(program("fixtures", &["run", "--timeout", "0"])); - assert_eq!(usage.status.code(), Some(2)); - } - fn timeout_kills_the_process_group() { let dir = tempfile::tempdir().expect("temp dir"); let pid_file = dir.path().join("pid"); + let (path, junit_arg) = junit(dir.path()); let mut command = program( "fixtures", - &["run", "hangs", "--timeout", "1", "--grace", "5"], + &[ + "run", + "hangs", + "--timeout", + "1", + "--grace", + "5", + "--junit", + &junit_arg, + ], ); command.env(PID_FILE_ENV, &pid_file); let started = Instant::now(); @@ -377,6 +389,8 @@ mod unix { ); assert!(stderr(&run).contains("TIMEOUT fixture::hangs: timed out after 1s")); assert_gone(read_pid(&pid_file)); + let xml = std::fs::read_to_string(path).expect("junit report"); + assert!(xml.contains(r#"type="timeout""#), "{xml}"); } fn sigterm_is_followed_by_sigkill() { @@ -402,7 +416,13 @@ mod unix { fn interrupt_stops_the_test_and_the_run() { let dir = tempfile::tempdir().expect("temp dir"); let pid_file = dir.path().join("pid"); - let mut command = program("fixtures", &["run", "hangs", "passes", "--grace", "5"]); + let (path, junit_arg) = junit(dir.path()); + let mut command = program( + "fixtures", + &[ + "run", "hangs", "passes", "--grace", "5", "--junit", &junit_arg, + ], + ); command .env(PID_FILE_ENV, &pid_file) .stdout(Stdio::piped()) @@ -421,6 +441,30 @@ mod unix { assert_eq!(run.status.code(), Some(2)); assert!(stderr(&run).contains("ERROR fixture::hangs: interrupted")); assert_gone(helper); + let xml = std::fs::read_to_string(path).expect("junit report"); + assert!(xml.contains(r#"type="interrupted""#), "{xml}"); + assert!(!xml.contains("fixture::passes"), "{xml}"); + } + + fn interrupt_between_tests_ends_the_run() { + let dir = tempfile::tempdir().expect("temp dir"); + let (path, junit_arg) = junit(dir.path()); + let run = output(program( + "fixtures", + &[ + "run", + "interrupts_after_exit", + "passes", + "--junit", + &junit_arg, + ], + )); + assert_eq!(run.status.code(), Some(2)); + assert!(stderr(&run).contains("<== PASS fixture::interrupts_after_exit")); + assert!(stderr(&run).contains("interrupted; 1 later test did not run")); + assert!(!stdout(&run).contains("passes: stdout")); + let xml = std::fs::read_to_string(path).expect("junit report"); + assert!(!xml.contains("fixture::passes"), "{xml}"); } fn a_passing_test_leaves_nothing_behind() { @@ -433,12 +477,51 @@ mod unix { assert_gone(read_pid(&pid_file)); } + /// Point the gateway lookup at a closed port, away from any registered + /// gateways. + fn without_gateway(mut command: Command, config: &Path) -> Command { + let closed = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let endpoint = format!("http://{}", closed.local_addr().expect("address")); + drop(closed); + command + .env("XDG_CONFIG_HOME", config) + .env("OPENSHELL_GATEWAY_ENDPOINT", endpoint) + .env_remove("OPENSHELL_GATEWAY") + .env("OPENSHELL_BIN", config.join("no-openshell")); + command + } + fn registered_tests_run_through_testkit() { let dir = tempfile::tempdir().expect("temp dir"); - let run = output(without_gateway(program("registered", &["run"]), dir.path())); + let (path, junit_arg) = junit(dir.path()); + let run = output(without_gateway( + program("registered", &["run", "--junit", &junit_arg]), + dir.path(), + )); assert_eq!(run.status.code(), Some(1)); // `testkit::run` logs the description and the gateway. assert!(stderr(&run).contains("[registered_needs_a_gateway] Needs a gateway.")); assert!(stderr(&run).contains("1 failed, 1 skipped")); + let xml = std::fs::read_to_string(path).expect("junit report"); + assert!( + xml.contains(r#""#), + "{xml}" + ); + } + + fn ready_fails_without_a_gateway() { + let dir = tempfile::tempdir().expect("temp dir"); + let started = Instant::now(); + let ready = output(without_gateway( + program("registered", &["ready", "--timeout", "1"]), + dir.path(), + )); + assert_eq!(ready.status.code(), Some(2)); + assert!(stderr(&ready).contains("not ready after 1s")); + assert!(started.elapsed() < Duration::from_secs(10)); } } From b01cfe0d3615d308a5a8412b53bb20639bf46220 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 4 Nov 2026 14:40:31 +0100 Subject: [PATCH 06/11] docs(testkit): document the test program and test registration Describe how `#[testkit::test]` registers tests, the `list`, `ready` and `run` subcommands, their exit codes, and the limits of `run`: Unix only, and a test stopped on a timeout does not run its cleanup. Signed-off-by: Evan Lezar --- tests/testkit/README.md | 63 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 1aea174049..3f786fe65d 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -7,7 +7,7 @@ This is a separate Cargo workspace with its own lockfile: | Crate | Purpose | |---|---| -| `testkit` | Finds and connects to the gateway, creates and cleans up sandboxes, runs the CLI. Also builds `testkit-ready`. | +| `testkit` | Finds and connects to the gateway, creates and cleans up sandboxes, runs the CLI, and runs test programs. Also builds `testkit-ready`. | | `testkit-macros` | The `#[testkit::test]` attribute. | ## Writing a test @@ -123,6 +123,67 @@ In a tmachine VM, the test runner (`tests/runner/`) does the same: it sets `OPENSHELL_BIN` to the installed CLI, then runs the suites with nextest. See [TESTING.md](../../TESTING.md). +## Test programs + +`#[testkit::test]` also registers the test with testkit. A binary whose +`main` calls `testkit::main()` runs every registered test it links, without +cargo or nextest: + +```rust +use conformance as _; + +fn main() -> std::process::ExitCode { + testkit::main() +} +``` + +The program must name each suite crate it runs, as `use conformance as _;` does +above: Rust doesn't link a dependency that is never named, and its tests would +be missing. A program that links no tests exits 2 with an error under every +command. `openshell-conformance`, a binary in the conformance crate, is the +program for the conformance suite; see +[its README](../suites/conformance/README.md). + +| Command | What it does | +|---|---| +| `list [FILTER]...` | Prints each test's name and description. Ignored tests are included and marked. | +| `ready [--timeout SECS]` | Waits for the gateway, like `testkit-ready` (default 60 seconds). Exits 2 if it is not ready in time. | +| `run [FILTER]...` | Runs the tests one at a time, each in its own process. Unix only. | + +A filter selects tests whose name contains it. `run` also takes: + +- `--include-ignored`: runs ignored tests too, instead of reporting them as + skipped. +- `--timeout SECS` (default 900): a test that runs longer gets SIGTERM, then + SIGKILL after `--grace SECS` (default 10). +- `--junit PATH`: writes a JUnit report with one test suite per family. + +Each test runs in its own process group. When the test's process exits, `run` +kills anything left in the group, such as CLI processes the test started. +Interrupting `run` stops the current test the same way and runs no more +tests; a second interrupt skips the grace period. If `run` itself is killed +with SIGKILL, the current test's process group keeps running. In a container +where `run` is PID 1, start it under an init, for example with +`docker run --init`. + +`run` exits 0 when every test passed, 1 when a test failed, timed out, or +crashed, and 2 when tests could not run: no tests matched, a test process +could not start or run its test, or the run was interrupted. + +A test that is stopped does not clean up: sandboxes it created stay on the +gateway. Run against a gateway you can throw away, or delete them by hand. + +Without a subcommand, the program takes libtest's arguments instead. A +`[[test]]` target with `harness = false` therefore also runs under `cargo test` +and nextest. In that mode tests run in one process, one at a time unless +`--test-threads` says otherwise. + +A test's name is its module path without the crate name, then its function +name, as in nextest. Its family, which groups tests in the JUnit report, is +the first directory under the last `suites` directory in the crate's path: +`conformance` for `tests/suites/conformance`, `features` for +`tests/suites/features/`. + ## Checks ```shell From 229a5312529d991cc2feff733486fca20ba3a8a2 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 2 Nov 2026 13:04:28 -0800 Subject: [PATCH 07/11] refactor(conformance): move the conformance suites into a library Cargo builds each file in tests/ as its own test executable, which no other program can link. Move the conformance suites into src/ as modules of a library, so a test program can link them. Test code is unchanged; cargo test and nextest run the same 15 tests as the library's unit tests. Nextest now names them as binary `conformance`, test `::`, so filters that selected a suite by binary select it by test name instead. Signed-off-by: Matthew Grossman Signed-off-by: Evan Lezar --- CI.md | 2 +- e2e/policy-advisor/README.md | 2 +- tests/runner/src/main.rs | 2 +- tests/suites/conformance/Cargo.toml | 2 +- tests/suites/conformance/README.md | 8 ++++---- tests/suites/conformance/{tests => src}/cli.rs | 0 .../conformance/{tests => src}/file_transfer.rs | 0 tests/suites/conformance/src/lib.rs | 16 ++++++++++++++++ .../conformance/{tests => src}/lifecycle.rs | 0 .../{tests => src}/persistent_workspace.rs | 0 .../conformance/{tests => src}/policy_advisor.rs | 0 tests/suites/conformance/{tests => src}/smoke.rs | 0 12 files changed, 24 insertions(+), 8 deletions(-) rename tests/suites/conformance/{tests => src}/cli.rs (100%) rename tests/suites/conformance/{tests => src}/file_transfer.rs (100%) create mode 100644 tests/suites/conformance/src/lib.rs rename tests/suites/conformance/{tests => src}/lifecycle.rs (100%) rename tests/suites/conformance/{tests => src}/persistent_workspace.rs (100%) rename tests/suites/conformance/{tests => src}/policy_advisor.rs (100%) rename tests/suites/conformance/{tests => src}/smoke.rs (100%) diff --git a/CI.md b/CI.md index ff278592e7..a1a1b8bc77 100644 --- a/CI.md +++ b/CI.md @@ -218,7 +218,7 @@ nextest filterset: nix run .#openshell-test-runner -- \ --environment ubuntu-docker-rootful --installer deb \ --family conformance --package conformance \ - --filter 'binary_id(=conformance::policy_advisor)' + --filter 'test(/^policy_advisor::/)' ``` A local run does not replace the required PR E2E gate. diff --git a/e2e/policy-advisor/README.md b/e2e/policy-advisor/README.md index c5c4e79ca0..adb700306f 100644 --- a/e2e/policy-advisor/README.md +++ b/e2e/policy-advisor/README.md @@ -69,7 +69,7 @@ test runner: nix run .#openshell-test-runner -- \ --environment ubuntu-docker-rootful --installer deb \ --family conformance --package conformance \ - --filter 'binary_id(=conformance::policy_advisor)' + --filter 'test(/^policy_advisor::/)' ``` The GitHub write test above and the regressions below still exercise distinct diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index 1503a4e0d7..9bac397824 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -38,7 +38,7 @@ struct Cli { #[arg(long = "package", short = 'p', value_name = "PACKAGE")] packages: Vec, /// A nextest filterset narrowing the tests that run, e.g. - /// `binary(smoke)`. + /// `test(/^smoke::/)`. #[arg(long, value_name = "FILTERSET")] filter: Option, /// The gateway name the installer registers with the CLI. The packaged diff --git a/tests/suites/conformance/Cargo.toml b/tests/suites/conformance/Cargo.toml index 40541d88c4..f40930d409 100644 --- a/tests/suites/conformance/Cargo.toml +++ b/tests/suites/conformance/Cargo.toml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# The conformance family is one crate; each file in `tests/` is a suite. +# The conformance family is one crate; each module in `src/` is a suite. [package] name = "conformance" description = "Conformance suite: sandbox behavior on every compute driver" diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index ee0502d072..88f4c8a2f7 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -2,7 +2,8 @@ The `conformance` crate holds the testkit suites for sandbox behavior that every compute driver must provide. The family is this one crate, its own workspace. -Each file in `tests/` is one suite. +It is a library: each module in `src/` is one suite, and `cargo test` and +nextest run the tests as the library's unit tests. | Suite | Covers | |---|---| @@ -28,9 +29,8 @@ gateway and in a tmachine VM, as CI does. ## Adding a test -Add a `#[testkit::test]` function to a suite, or a new file in `tests/` for a new -area. No playbook, test archive or workflow entry is needed: the runner builds -and runs every test in the crate. +Add a `#[testkit::test]` function to a suite, or for a new area, a new file in +`src/` and a `mod` line for it in `src/lib.rs`. ## What a test may assert diff --git a/tests/suites/conformance/tests/cli.rs b/tests/suites/conformance/src/cli.rs similarity index 100% rename from tests/suites/conformance/tests/cli.rs rename to tests/suites/conformance/src/cli.rs diff --git a/tests/suites/conformance/tests/file_transfer.rs b/tests/suites/conformance/src/file_transfer.rs similarity index 100% rename from tests/suites/conformance/tests/file_transfer.rs rename to tests/suites/conformance/src/file_transfer.rs diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs new file mode 100644 index 0000000000..728c138b9e --- /dev/null +++ b/tests/suites/conformance/src/lib.rs @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Conformance suite: sandbox behavior that every compute driver must +//! provide. Each module is one suite. +//! +//! `cargo test` and nextest run the tests as this library's unit tests. +//! `#[testkit::test]` also registers them, so a test program runs them by +//! linking this crate. + +mod cli; +mod file_transfer; +mod lifecycle; +mod persistent_workspace; +mod policy_advisor; +mod smoke; diff --git a/tests/suites/conformance/tests/lifecycle.rs b/tests/suites/conformance/src/lifecycle.rs similarity index 100% rename from tests/suites/conformance/tests/lifecycle.rs rename to tests/suites/conformance/src/lifecycle.rs diff --git a/tests/suites/conformance/tests/persistent_workspace.rs b/tests/suites/conformance/src/persistent_workspace.rs similarity index 100% rename from tests/suites/conformance/tests/persistent_workspace.rs rename to tests/suites/conformance/src/persistent_workspace.rs diff --git a/tests/suites/conformance/tests/policy_advisor.rs b/tests/suites/conformance/src/policy_advisor.rs similarity index 100% rename from tests/suites/conformance/tests/policy_advisor.rs rename to tests/suites/conformance/src/policy_advisor.rs diff --git a/tests/suites/conformance/tests/smoke.rs b/tests/suites/conformance/src/smoke.rs similarity index 100% rename from tests/suites/conformance/tests/smoke.rs rename to tests/suites/conformance/src/smoke.rs From 42dae7ee7510325a3b79357c71c8aee094aa816f Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 2 Nov 2026 10:31:09 -0800 Subject: [PATCH 08/11] feat(conformance): add the openshell-conformance program The conformance crate now also builds openshell-conformance, a binary whose main calls testkit::main() and links the crate's library. It lists, waits for the gateway, and runs the conformance tests without cargo or nextest. It runs only the conformance suite and promises only the conformance contract; feature and driver suites are not part of it. The binary has test = false, so cargo test and nextest still see one conformance test binary, the library's unit tests, and the e2e lanes and the runner's filters are unchanged. The library is linked without cfg(test), so the program gets the registrations and none of the tokio test wrappers. tasks/scripts/check-conformance-program.sh checks that openshell-conformance list names the same tests as nextest; Branch Checks and mise test:rust run it. A dependency that is never named is not linked, so a suite the program forgets to name would silently go missing. testkit::main() now fails with exit 2 when a program links no tests, under every command. Signed-off-by: Matthew Grossman --- .github/workflows/branch-checks.yml | 1 + TESTING.md | 6 +++++ tasks/scripts/check-conformance-program.sh | 30 ++++++++++++++++++++++ tasks/test.toml | 1 + tests/suites/conformance/Cargo.toml | 8 ++++++ tests/suites/conformance/README.md | 25 ++++++++++++++++-- tests/suites/conformance/src/lib.rs | 4 +-- tests/suites/conformance/src/main.rs | 18 +++++++++++++ 8 files changed, 89 insertions(+), 4 deletions(-) create mode 100755 tasks/scripts/check-conformance-program.sh create mode 100644 tests/suites/conformance/src/main.rs diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index 8e21eb7a69..db22a74913 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -392,6 +392,7 @@ jobs: cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/tmachine/Cargo.toml cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/runner/Cargo.toml cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/support/Cargo.toml + tasks/scripts/check-conformance-program.sh rust-postgres-test: name: Rust PostgreSQL tests diff --git a/TESTING.md b/TESTING.md index 5e74f53c72..0503c2bb4e 100644 --- a/TESTING.md +++ b/TESTING.md @@ -181,6 +181,12 @@ by `OPENSHELL_BIN`. See [tests/testkit/README.md](tests/testkit/README.md) for how to write and run tests. +The conformance crate also builds `openshell-conformance`, a program that +runs the conformance suite against a gateway without cargo or nextest +(`openshell-conformance list`, `ready`, `run`). It covers only the +conformance suite. See +[tests/suites/conformance/README.md](tests/suites/conformance/README.md). + Suites live under `tests/suites/`, in Cargo workspaces with their own lockfiles: `conformance`, `features`, and one workspace per driver suite under `drivers/`. Run the `conformance` suites against a local Docker diff --git a/tasks/scripts/check-conformance-program.sh b/tasks/scripts/check-conformance-program.sh new file mode 100755 index 0000000000..3284f20418 --- /dev/null +++ b/tasks/scripts/check-conformance-program.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Check that `openshell-conformance list` names the same tests that nextest +# lists for the conformance crate, so the program runs the whole suite. +# Neither command needs a gateway. + +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +manifest="${root}/tests/suites/conformance/Cargo.toml" +work="$(mktemp -d)" +trap 'rm -rf "${work}"' EXIT + +cargo run --quiet --locked --manifest-path "${manifest}" --bin openshell-conformance -- list \ + | awk '{print $1}' | sort > "${work}/program" +cargo nextest list --locked --color never --manifest-path "${manifest}" --message-format oneline \ + | awk '{print $2}' | sort > "${work}/nextest" + +if ! diff -u "${work}/nextest" "${work}/program"; then + echo "error: openshell-conformance list differs from nextest's list (-nextest +program)" >&2 + exit 1 +fi +if [[ ! -s "${work}/program" ]]; then + echo "error: openshell-conformance lists no tests" >&2 + exit 1 +fi +echo "openshell-conformance lists the same $(wc -l < "${work}/program" | tr -d ' ') tests as nextest" diff --git a/tasks/test.toml b/tasks/test.toml index 4309ad95b7..2839c1cf0c 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -134,6 +134,7 @@ run = [ "cargo nextest run --config-file .config/nextest.toml --manifest-path tests/tmachine/Cargo.toml", "cargo nextest run --config-file .config/nextest.toml --manifest-path tests/runner/Cargo.toml", "cargo nextest run --config-file .config/nextest.toml --manifest-path tests/support/Cargo.toml", + "tasks/scripts/check-conformance-program.sh", ] run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test-precommit native" hide = true diff --git a/tests/suites/conformance/Cargo.toml b/tests/suites/conformance/Cargo.toml index f40930d409..99274d8dfa 100644 --- a/tests/suites/conformance/Cargo.toml +++ b/tests/suites/conformance/Cargo.toml @@ -12,6 +12,14 @@ license = "Apache-2.0" repository = "https://github.com/NVIDIA/OpenShell" publish = false +# `openshell-conformance`: the suite as one program to run against a gateway. +# No test target of its own: cargo test and nextest run the tests through the +# library. +[[bin]] +name = "openshell-conformance" +path = "src/main.rs" +test = false + [dependencies] openshell-testkit = { path = "../../testkit/testkit" } serde_json = "1" diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index 88f4c8a2f7..fe5c95b5f8 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -3,7 +3,8 @@ The `conformance` crate holds the testkit suites for sandbox behavior that every compute driver must provide. The family is this one crate, its own workspace. It is a library: each module in `src/` is one suite, and `cargo test` and -nextest run the tests as the library's unit tests. +nextest run the tests as the library's unit tests. The crate also builds the +`openshell-conformance` program, which runs the same tests. | Suite | Covers | |---|---| @@ -27,10 +28,30 @@ environments). [TESTING.md](../../../TESTING.md) covers running it against a local Docker gateway and in a tmachine VM, as CI does. +### The `openshell-conformance` program + +`openshell-conformance` runs the suite against a gateway without cargo or +nextest, so a driver author or operator can check their own gateway. It finds +the gateway and the CLI as the tests do. Build it and run it with: + +```shell +cargo build --locked --manifest-path tests/suites/conformance/Cargo.toml --bin openshell-conformance +tests/suites/conformance/target/debug/openshell-conformance list +tests/suites/conformance/target/debug/openshell-conformance ready +tests/suites/conformance/target/debug/openshell-conformance run --junit conformance.xml +``` + +The program runs only this suite: passing it means a gateway meets the +conformance contract below. Feature and driver suites are not part of it. The +[testkit README](../../testkit/README.md#test-programs) describes the commands +and exit codes. `tasks/scripts/check-conformance-program.sh` checks that it +lists the same tests as nextest. + ## Adding a test Add a `#[testkit::test]` function to a suite, or for a new area, a new file in -`src/` and a `mod` line for it in `src/lib.rs`. +`src/` and a `mod` line for it in `src/lib.rs`. `openshell-conformance` +includes every test in the crate. ## What a test may assert diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 728c138b9e..00a4d45027 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -5,8 +5,8 @@ //! provide. Each module is one suite. //! //! `cargo test` and nextest run the tests as this library's unit tests. -//! `#[testkit::test]` also registers them, so a test program runs them by -//! linking this crate. +//! `#[testkit::test]` also registers them, so the `openshell-conformance` +//! program in `src/main.rs` runs them by linking this library. mod cli; mod file_transfer; diff --git a/tests/suites/conformance/src/main.rs b/tests/suites/conformance/src/main.rs new file mode 100644 index 0000000000..7b4e899c60 --- /dev/null +++ b/tests/suites/conformance/src/main.rs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `openshell-conformance`: the conformance suite as one program, to run +//! against a gateway without cargo or nextest. +//! +//! The tests are registered by `#[testkit::test]` in this crate's library; +//! see `testkit::program` for the commands. + +use std::process::ExitCode; + +// A dependency that is never named is not linked, and its tests would be +// missing. +use conformance as _; + +fn main() -> ExitCode { + testkit::main() +} From 75af0a18afb9328568864d2add60f96b7d061828 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 3 Nov 2026 13:11:32 -0800 Subject: [PATCH 09/11] test(e2e): run the conformance host lanes with openshell-conformance The Docker, Podman, VM and Kubernetes host lanes ran the conformance suite with cargo nextest. They now run it with openshell-conformance: ready, then run --junit, so CI exercises the program that gateway owners use. The script uses OPENSHELL_CONFORMANCE_BIN when set and otherwise builds the program from the checkout. The JUnit report goes to OPENSHELL_CONFORMANCE_JUNIT or results/conformance-.xml; the Kubernetes runner names it after its report so credential-driver runs keep both. Signed-off-by: Matthew Grossman --- TESTING.md | 4 +++- e2e/rust/e2e-kubernetes.sh | 3 ++- e2e/support/conformance.sh | 45 +++++++++++++++++++++++++++++--------- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/TESTING.md b/TESTING.md index 0503c2bb4e..cd1062b391 100644 --- a/TESTING.md +++ b/TESTING.md @@ -425,7 +425,9 @@ Suites: - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests. - Conformance (`tests/suites/conformance`) - the testkit `conformance` crate: lifecycle, the CLI, persistent workspaces, file transfer, and the policy - advisor. Driver E2E runs it with nextest against the lane's gateway. The + advisor. Driver E2E runs it with `openshell-conformance` against the lane's + gateway and writes a JUnit report under `results/`; set + `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program. The installed-artifact conformance lanes run the same crate through `openshell-test-runner` (see above). - Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index 9bdcf4e6a2..fc90a827ee 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -79,7 +79,8 @@ run_conformance() { return 0 fi - e2e_run_openshell_conformance "Kubernetes" + OPENSHELL_CONFORMANCE_JUNIT="${OPENSHELL_CONFORMANCE_JUNIT:-${ROOT}/results/${OPENSHELL_E2E_REPORT_NAME:-e2e-kubernetes}-conformance.xml}" \ + e2e_run_openshell_conformance "Kubernetes" } # `OPENSHELL_E2E_REPORT_NAME` (default `e2e-kubernetes`) names the report diff --git a/e2e/support/conformance.sh b/e2e/support/conformance.sh index 26cb3ed9f6..82594439c0 100644 --- a/e2e/support/conformance.sh +++ b/e2e/support/conformance.sh @@ -16,30 +16,55 @@ e2e_require_openshell_bin() { return 2 fi - # Cargo runs each test from its package directory, so resolve the candidate - # relative to the caller's working directory before passing it to the tests. + # Tests may run from another directory, so resolve the candidate relative + # to the caller's working directory before passing it to them. local binary_dir binary_dir="$(cd "$(dirname "${OPENSHELL_BIN}")" && pwd -P)" || return export OPENSHELL_BIN="${binary_dir}/$(basename "${OPENSHELL_BIN}")" } +# Print the path of the openshell-conformance program: OPENSHELL_CONFORMANCE_BIN +# if set, else a debug build from this checkout. +e2e_openshell_conformance_bin() { + local root=$1 + + if [ -n "${OPENSHELL_CONFORMANCE_BIN:-}" ]; then + if [ ! -x "${OPENSHELL_CONFORMANCE_BIN}" ]; then + echo "ERROR: openshell-conformance is not executable: ${OPENSHELL_CONFORMANCE_BIN}" >&2 + return 2 + fi + echo "${OPENSHELL_CONFORMANCE_BIN}" + return + fi + + cargo build \ + --locked \ + --manifest-path "${root}/tests/suites/conformance/Cargo.toml" \ + --bin openshell-conformance >&2 || return + # Cargo resolves a relative CARGO_TARGET_DIR against the working directory. + local target_dir + target_dir="$(cd "${CARGO_TARGET_DIR:-${root}/tests/suites/conformance/target}" && pwd)" || return + echo "${target_dir}/debug/openshell-conformance" +} + +# Run the conformance suite with openshell-conformance. The JUnit report goes +# to OPENSHELL_CONFORMANCE_JUNIT, else results/conformance-.xml. e2e_run_openshell_conformance() { local gateway_label=${1:-OpenShell} - local root + local root program junit root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" e2e_require_openshell_bin || return + program="$(e2e_openshell_conformance_bin "${root}")" || return + junit="${OPENSHELL_CONFORMANCE_JUNIT:-${root}/results/conformance-$(echo "${gateway_label}" | tr '[:upper:]' '[:lower:]').xml}" + mkdir -p "$(dirname "${junit}")" || return - # The tests find the gateway as the CLI does: OPENSHELL_GATEWAY_ENDPOINT, + # The program finds the gateway as the CLI does: OPENSHELL_GATEWAY_ENDPOINT, # else OPENSHELL_GATEWAY, else the active gateway. The gateway wrappers set # these for the gateway they started. echo "==> Running conformance suites against the ${gateway_label} gateway" - cargo nextest run \ - --locked \ - --manifest-path "${root}/tests/suites/conformance/Cargo.toml" \ - --test-threads 1 \ - --no-fail-fast \ - --no-tests fail + "${program}" ready || return + "${program}" run --junit "${junit}" } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then From 1d0f23c29e1495af39f4808ae6393b6770b4dca7 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 3 Nov 2026 13:16:51 -0800 Subject: [PATCH 10/11] ci(e2e): build openshell-conformance once per architecture in Branch E2E Each Docker, Podman, VM and Kubernetes host lane built the conformance suite itself. Branch E2E now builds openshell-conformance once per architecture as a static musl binary, checks that it is static and lists its tests, and uploads it. setup-e2e-cli downloads the one that matches the runner when a lane sets prebuilt-conformance, and the lane runs it through OPENSHELL_CONFORMANCE_BIN. The job is separate from build-binaries.yml, which the release workflows also call: the program has its own workspace and is not released. Lanes called without prebuilt-conformance, such as the release qualification lanes, still build the program from the checkout. Signed-off-by: Matthew Grossman --- .github/actions/setup-e2e-cli/action.yml | 22 +++++ .github/workflows/branch-e2e.yml | 100 ++++++++++++++++++---- .github/workflows/e2e-docker-test.yml | 6 ++ .github/workflows/e2e-kubernetes-test.yml | 7 ++ .github/workflows/e2e-podman-test.yml | 6 ++ .github/workflows/e2e-vm-test.yml | 6 ++ TESTING.md | 3 +- 7 files changed, 134 insertions(+), 16 deletions(-) diff --git a/.github/actions/setup-e2e-cli/action.yml b/.github/actions/setup-e2e-cli/action.yml index 8b30970d8d..6bc2a78ec2 100644 --- a/.github/actions/setup-e2e-cli/action.yml +++ b/.github/actions/setup-e2e-cli/action.yml @@ -1,6 +1,12 @@ name: Setup E2E CLI description: Download architecture-matched prebuilt OpenShell host CLIs for E2E tests +inputs: + conformance: + description: Also download the prebuilt openshell-conformance program and set OPENSHELL_CONFORMANCE_BIN + required: false + default: "false" + runs: using: composite steps: @@ -17,3 +23,19 @@ runs: chmod +x "$cli" "$cli" --version echo "OPENSHELL_BIN=$cli" >> "$GITHUB_ENV" + + - name: Download prebuilt openshell-conformance + if: inputs.conformance == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ runner.arch == 'X64' && 'openshell-conformance-x86_64-unknown-linux-musl' || 'openshell-conformance-aarch64-unknown-linux-musl' }} + path: .e2e/prebuilt-conformance + + - name: Configure prebuilt openshell-conformance + if: inputs.conformance == 'true' + shell: bash + run: | # zizmor: ignore[github-env] persists only a fixed path under the trusted workspace + program="$GITHUB_WORKSPACE/.e2e/prebuilt-conformance/openshell-conformance" + chmod +x "$program" + "$program" list + echo "OPENSHELL_CONFORMANCE_BIN=$program" >> "$GITHUB_ENV" diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index cc558fb4e2..0f59ab2a81 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -144,6 +144,60 @@ jobs: cargo-version: ${{ needs.version.outputs.cargo }} extra-cargo-flags: --no-default-features + # The conformance program, built once per architecture for the host lanes. + # It lives in its own workspace and is not released, so it is not part of + # build-binaries.yml. + build-conformance: + name: openshell-conformance (${{ matrix.triple }}) + needs: [pr_metadata] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_any_e2e == 'true' + permissions: + contents: read + strategy: + matrix: + include: + - triple: x86_64-unknown-linux-musl + runner: linux-amd64-cpu8 + - triple: aarch64-unknown-linux-musl + runner: linux-arm64-cpu8 + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.sha }} + - uses: ./.github/actions/setup-nix + with: + prepare-shell: "true" + cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} + - uses: ./.github/actions/setup-rust + with: + cache-key: binaries-v2-openshell-conformance-${{ matrix.triple }} + save-cache: ${{ github.ref == 'refs/heads/main' }} + - name: Build openshell-conformance + shell: nix develop -c bash -euo pipefail {0} + env: + # Build into the root target directory, which setup-rust caches. + CARGO_TARGET_DIR: ${{ github.workspace }}/target + TRIPLE: ${{ matrix.triple }} + run: | + cargo build --release --locked --target "$TRIPLE" \ + --manifest-path tests/suites/conformance/Cargo.toml \ + --bin openshell-conformance + binary="target/$TRIPLE/release/openshell-conformance" + # The program must run on any lane host. + tasks/scripts/verify-static-binary.sh "$binary" + # Fails if the program has no tests linked. + "$binary" list + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: openshell-conformance-${{ matrix.triple }} + path: target/${{ matrix.triple }}/release/openshell-conformance + compression-level: 0 + retention-days: 5 + if-no-files-found: error + build-external-drivers: name: ${{ matrix.binary }} (${{ matrix.triple }}) needs: [pr_metadata, version] @@ -306,7 +360,7 @@ jobs: ] docker-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -314,11 +368,12 @@ jobs: packages: read uses: ./.github/workflows/e2e-docker-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} runner: linux-arm64-cpu8 podman-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -326,12 +381,13 @@ jobs: packages: read uses: ./.github/workflows/e2e-podman-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} suite-matrix: >- [{"suite":"ci","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:ci"}] podman-additional-ca-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -339,21 +395,24 @@ jobs: packages: read uses: ./.github/workflows/e2e-podman-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} suite-matrix: >- [{"suite":"additional-ca","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"OPENSHELL_E2E_PODMAN_TEST=additional_ca mise run --no-deps --skip-deps e2e:podman"}] vm-e2e: - needs: [pr_metadata, build-binaries, build-vm-driver] + needs: [pr_metadata, build-binaries, build-conformance, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read contents: read packages: read uses: ./.github/workflows/e2e-vm-test.yml + with: + prebuilt-conformance: true vm-additional-ca-e2e: - needs: [pr_metadata, build-binaries, build-vm-driver] + needs: [pr_metadata, build-binaries, build-conformance, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -361,11 +420,12 @@ jobs: packages: read uses: ./.github/workflows/e2e-vm-test.yml with: + prebuilt-conformance: true suite-name: additional-ca test-name: additional_ca docker-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-external-drivers, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -373,6 +433,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-docker-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} runner: linux-arm64-cpu8 gateway-artifact: openshell-gateway-plain-aarch64-unknown-linux-gnu @@ -381,7 +442,7 @@ jobs: [{"suite":"external-driver","cmd":"mise run --no-deps --skip-deps e2e:docker:external-driver","apt_packages":"openssh-client","python_proto":false,"mcp":false}] podman-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-external-drivers, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -389,6 +450,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-podman-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu external-driver-binary: openshell-driver-podman @@ -396,7 +458,7 @@ jobs: [{"suite":"external-driver","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:external-driver"}] vm-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-vm-driver] + needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -404,6 +466,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-vm-test.yml with: + prebuilt-conformance: true gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu suite-name: external-driver e2e-task: e2e:vm:external-driver @@ -420,7 +483,7 @@ jobs: image-tag: ${{ github.sha }} kubernetes-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' strategy: fail-fast: false @@ -444,13 +507,14 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: ${{ matrix.job_name }} agent-sandbox-version: ${{ matrix.agent_sandbox_version }} test-name: ${{ matrix.test_name }} kubernetes-app-listener-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' strategy: fail-fast: false @@ -478,6 +542,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes app listener (${{ matrix.name }}) e2e-task: e2e:kubernetes:app-listener @@ -487,7 +552,7 @@ jobs: listener-takeover: ${{ matrix.takeover }} kubernetes-workspace-managed-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -495,12 +560,13 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes E2E (workspace managed mode) e2e-task: e2e:kubernetes:workspace-managed kubernetes-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-external-drivers, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -508,6 +574,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes E2E (external compute driver) e2e-task: e2e:kubernetes:external-driver @@ -516,7 +583,7 @@ jobs: cluster-images: sandbox supervisor kubernetes-workspace-operator-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -524,12 +591,13 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes E2E (workspace operator mode) e2e-task: e2e:kubernetes:workspace-operator kubernetes-ha-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' permissions: actions: read @@ -537,6 +605,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes HA E2E (Rust smoke) extra-helm-values: deploy/helm/openshell/ci/values-high-availability.yaml @@ -546,7 +615,7 @@ jobs: use-envoy-gateway: true kubernetes-credential-drivers-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true' permissions: actions: read @@ -554,6 +623,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes Credential Drivers E2E e2e-task: e2e:kubernetes:credential-drivers diff --git a/.github/workflows/e2e-docker-test.yml b/.github/workflows/e2e-docker-test.yml index 501ded8d34..faa0121cd2 100644 --- a/.github/workflows/e2e-docker-test.yml +++ b/.github/workflows/e2e-docker-test.yml @@ -21,6 +21,10 @@ on: required: false type: string default: "" + prebuilt-conformance: + required: false + type: boolean + default: false external-driver-binary: required: false type: string @@ -67,6 +71,8 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-e2e-cli + with: + conformance: ${{ inputs.prebuilt-conformance }} - uses: ./.github/actions/setup-e2e-gateway with: diff --git a/.github/workflows/e2e-kubernetes-test.yml b/.github/workflows/e2e-kubernetes-test.yml index 867b8c1dd7..1ae616e36e 100644 --- a/.github/workflows/e2e-kubernetes-test.yml +++ b/.github/workflows/e2e-kubernetes-test.yml @@ -67,6 +67,11 @@ on: required: false type: string default: "" + prebuilt-conformance: + description: "Run conformance with the openshell-conformance artifact instead of building it" + required: false + type: boolean + default: false external-driver-binary: description: "Optional standalone compute driver binary used to compose a local test image" required: false @@ -124,6 +129,8 @@ jobs: - name: Use prebuilt OpenShell CLI uses: ./.github/actions/setup-e2e-cli + with: + conformance: ${{ inputs.prebuilt-conformance }} - name: Use prebuilt OpenShell gateway if: inputs.gateway-artifact != '' diff --git a/.github/workflows/e2e-podman-test.yml b/.github/workflows/e2e-podman-test.yml index d70ad6c60c..ac6d7d66ef 100644 --- a/.github/workflows/e2e-podman-test.yml +++ b/.github/workflows/e2e-podman-test.yml @@ -17,6 +17,10 @@ on: required: false type: string default: "" + prebuilt-conformance: + required: false + type: boolean + default: false external-driver-binary: required: false type: string @@ -52,6 +56,8 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-e2e-cli + with: + conformance: ${{ inputs.prebuilt-conformance }} - uses: ./.github/actions/setup-e2e-gateway with: diff --git a/.github/workflows/e2e-vm-test.yml b/.github/workflows/e2e-vm-test.yml index 28b4c81db5..82d86058b8 100644 --- a/.github/workflows/e2e-vm-test.yml +++ b/.github/workflows/e2e-vm-test.yml @@ -14,6 +14,10 @@ on: required: false type: string default: "" + prebuilt-conformance: + required: false + type: boolean + default: false suite-name: required: false type: string @@ -47,6 +51,8 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-e2e-cli + with: + conformance: ${{ inputs.prebuilt-conformance }} - uses: ./.github/actions/setup-e2e-gateway with: diff --git a/TESTING.md b/TESTING.md index cd1062b391..db37b757c4 100644 --- a/TESTING.md +++ b/TESTING.md @@ -427,7 +427,8 @@ Suites: lifecycle, the CLI, persistent workspaces, file transfer, and the policy advisor. Driver E2E runs it with `openshell-conformance` against the lane's gateway and writes a JUnit report under `results/`; set - `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program. The + `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program, as Branch E2E does + with the static build it makes once per architecture. The installed-artifact conformance lanes run the same crate through `openshell-test-runner` (see above). - Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or From 43924947d4249e77e12c1c32a9eed18f690ebee4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 4 Nov 2026 20:38:32 -0800 Subject: [PATCH 11/11] test(e2e): harden the conformance program helpers Take the openshell-conformance path from cargo's build messages instead of assuming /debug, so CARGO_BUILD_TARGET and other profiles work. Make the drift check fail clearly when it reads no test names from nextest. Signed-off-by: Matthew Grossman --- e2e/support/conformance.sh | 19 +++++++++++++------ tasks/scripts/check-conformance-program.sh | 4 ++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/e2e/support/conformance.sh b/e2e/support/conformance.sh index 82594439c0..56ce5a50f1 100644 --- a/e2e/support/conformance.sh +++ b/e2e/support/conformance.sh @@ -37,14 +37,21 @@ e2e_openshell_conformance_bin() { return fi - cargo build \ + # Cargo reports where it put the binary, which depends on the target + # directory and on CARGO_BUILD_TARGET. Diagnostics still go to stderr. + local messages program + messages="$(cargo build \ --locked \ + --message-format=json-render-diagnostics \ --manifest-path "${root}/tests/suites/conformance/Cargo.toml" \ - --bin openshell-conformance >&2 || return - # Cargo resolves a relative CARGO_TARGET_DIR against the working directory. - local target_dir - target_dir="$(cd "${CARGO_TARGET_DIR:-${root}/tests/suites/conformance/target}" && pwd)" || return - echo "${target_dir}/debug/openshell-conformance" + --bin openshell-conformance)" || return + program="$(printf '%s\n' "${messages}" \ + | sed -n 's/.*"executable":"\([^"]*\)".*/\1/p' | tail -n 1)" + if [ -z "${program}" ] || [ ! -x "${program}" ]; then + echo "ERROR: cargo did not report the openshell-conformance binary" >&2 + return 2 + fi + echo "${program}" } # Run the conformance suite with openshell-conformance. The JUnit report goes diff --git a/tasks/scripts/check-conformance-program.sh b/tasks/scripts/check-conformance-program.sh index 3284f20418..7bfb2ac8c4 100755 --- a/tasks/scripts/check-conformance-program.sh +++ b/tasks/scripts/check-conformance-program.sh @@ -19,6 +19,10 @@ cargo run --quiet --locked --manifest-path "${manifest}" --bin openshell-conform cargo nextest list --locked --color never --manifest-path "${manifest}" --message-format oneline \ | awk '{print $2}' | sort > "${work}/nextest" +if [[ ! -s "${work}/nextest" ]]; then + echo "error: could not read test names from nextest's list; has its oneline format changed?" >&2 + exit 1 +fi if ! diff -u "${work}/nextest" "${work}/program"; then echo "error: openshell-conformance list differs from nextest's list (-nextest +program)" >&2 exit 1