stackinator 7: spack version 1.2 support

docs/building.md

@@ -13,11 +13,24 @@ stack-config --build $BUILD_PATH ...
 
 # perform the build
 cd $BUILD_PATH
-env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make modules store.squashfs -j32
+env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make modules store.squashfs NJOBS=32
 ```
 
 The call to `make` is wrapped with with `env --ignore-env` to unset all environment variables, to improve reproducability of builds.
 
+## Controlling build parallelism
+
+The number of packages and compilation jobs that Spack runs concurrently is set with the `NJOBS` make variable:
+
+```
+make store.squashfs NJOBS=64
+```
+
+Set `NJOBS` to the number of cores available on the build node (it defaults to `16` if not provided).
+
+!!! note
+    Do not use `make -jN` to control build parallelism. The `install` step clears `MAKEFLAGS` before invoking Spack — this avoids a crash with GNU make older than 4.4, whose legacy file-descriptor jobserver Spack mishandles — so the outer `make -j` flag does not reach Spack. `NJOBS` is passed to Spack as `spack install --jobs`, and is the only flag that governs build parallelism.
+
 Build times for stacks typically vary between 30 minutes to 3 hours, depending on the specific packages that have to be built.
 Using [build caches][ref-mirrors] and building in shared memory (see below) are the most effective methods to speed up builds.
 

stackinator/etc/Make.inc

@@ -1,8 +1,22 @@
 # vi: filetype=make
 
+# Suppress make's recipe echo. Sandboxed commands self-label via the sandbox
+# wrapper, and build phases are announced by the banner macro below.
+.SILENT:
+
 SPACK ?= spack
+SPACK_HELPER := $(SPACK) --color=never
+
+# Per-phase banners. Colored on a terminal (MAKE_TERMOUT), plain when piped.
+ifeq ($(origin NO_COLOR),undefined)
+ifneq ($(MAKE_TERMOUT),)
+BANNER_COLOR := \033[1;93m
+BANNER_RESET := \033[0m
+endif
+endif
 
-SPACK_ENV = $(SPACK) -e $(dir $@)
+# Usage: $(call banner,Human readable phase name)
+banner = @printf '\n%b==> [stackinator] %s%b\n' '$(BANNER_COLOR)' '$(1)' '$(BANNER_RESET)'
 
 ifndef STORE
 $(error STORE should point to a Spack install root)
@@ -14,19 +28,3 @@ endif
 
 store:
 	mkdir -p $(STORE)
-
-# Concretization
-%/spack.lock: %/spack.yaml %/config.yaml %/packages.yaml
-	/usr/bin/time -f '"%C" took %e seconds.' $(SPACK_ENV) concretize -f
-
-# Generate Makefiles for the environment install
-%/Makefile: %/spack.lock
-	$(SPACK_ENV) env depfile --make-target-prefix $*/generated -o $@
-
-# For generating {compilers,config,packages}.yaml files.
-%.yaml: export SPACK_USER_CONFIG_PATH=$(abspath $(dir $@))
-%.yaml:
-	touch $@
-
-# Because Spack doesn't know how to find compilers, we help it by getting the bin folder of gcc, clang, nvc given a install prefix
-compiler_bin_dirs = $$(find $(1) '(' -name gcc -o -name clang -o -name nvc -o -name icx ')' -path '*/bin/*' '(' -type f -o -type l ')' -exec dirname {} +)

stackinator/etc/compiler-config.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""
+Generate or update a packages.yaml file with compiler externals derived from
+installed Spack packages. Intended to be run as:
+
+    spack -e BUILD_ROOT python compiler-config.py OUTPUT_YAML COMPILER [COMPILER ...]
+
+If OUTPUT_YAML already exists (e.g. the build packages.yaml), the compiler
+entries are merged in rather than replacing existing content.
+"""
+
+import argparse
+import os
+import sys
+
+import yaml
+
+
+_COMPILER_BINS = {
+    "gcc": [("gcc", "c"), ("g++", "cxx"), ("gfortran", "fortran")],
+    "llvm": [("clang", "c"), ("clang++", "cxx"), ("flang-new", "fortran")],
+    "llvm-amdgpu": [("clang", "c"), ("clang++", "cxx"), ("flang-new", "fortran")],
+    "nvhpc": [("nvc", "c"), ("nvc++", "cxx"), ("nvfortran", "fortran")],
+    "intel-oneapi-compilers": [("icx", "c"), ("icpx", "cxx"), ("ifx", "fortran")],
+}
+
+
+def find_compiler_bins(prefix, compiler_name):
+    """
+    Return a dict mapping language keys (c, cxx, fortran) to absolute binary
+    paths found under prefix, or None if nothing was found.
+    """
+    candidates = _COMPILER_BINS.get(compiler_name, [])
+    result = {}
+    for root, _dirs, files in os.walk(prefix):
+        for exe, lang in candidates:
+            if lang not in result and exe in files:
+                full = os.path.join(root, exe)
+                if os.access(full, os.X_OK):
+                    result[lang] = full
+        if len(result) == len(candidates):
+            break
+    return result or None
+
+
+def build_compiler_packages(compiler_names):
+    """
+    Query the active Spack DB for each compiler name and return a dict
+    suitable for merging into packages.yaml.
+    """
+    import spack.store
+
+    packages = {}
+    for name in compiler_names:
+        specs = list(spack.store.STORE.db.query(name, explicit=False))
+        if not specs:
+            print(f"  compiler-config: no installed specs found for '{name}'", file=sys.stderr)
+            continue
+
+        externals = []
+        for spec in specs:
+            # Skip externals such as the system gcc. It is registered in the
+            # cluster packages.yaml and gets pulled into the DB as a bootstrap
+            # dependency, but only compilers actually built into this environment
+            # should be surfaced here. The system compiler is included separately,
+            # and only when selected, via --system-packages.
+            if spec.external:
+                continue
+            prefix = str(spec.prefix)
+            bins = find_compiler_bins(prefix, name)
+            if not bins:
+                print(f"  compiler-config: no binaries found for {name} at {prefix}", file=sys.stderr)
+                continue
+            externals.append(
+                {
+                    "spec": f"{spec.name}@{spec.version}",
+                    "prefix": prefix,
+                    "extra_attributes": {"compilers": bins},
+                }
+            )
+            print(f"  compiler-config: found {name}@{spec.version} at {prefix}", file=sys.stderr)
+
+        if externals:
+            packages[name] = {"externals": externals, "buildable": False}
+
+    return packages
+
+
+def load_system_compiler_externals(system_packages_path):
+    """
+    Read a packages.yaml and return entries that carry extra_attributes.compilers.
+    Used to surface system (external) compilers such as system gcc into the output.
+    """
+    with open(system_packages_path) as fid:
+        data = yaml.safe_load(fid) or {}
+
+    packages = {}
+    for pkg_name, pkg_data in data.get("packages", {}).items():
+        if not isinstance(pkg_data, dict):
+            continue
+        compiler_externals = [
+            e
+            for e in pkg_data.get("externals", [])
+            if isinstance(e, dict) and "extra_attributes" in e and "compilers" in e["extra_attributes"]
+        ]
+        if compiler_externals:
+            packages[pkg_name] = {"externals": compiler_externals, "buildable": False}
+            print(f"  compiler-config: found system compiler '{pkg_name}' in {system_packages_path}", file=sys.stderr)
+
+    return packages
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("output", help="Path to packages.yaml to create or update")
+    parser.add_argument("compilers", nargs="*", help="Compiler package names to query")
+    parser.add_argument(
+        "--system-packages",
+        help="Path to a packages.yaml to read system compiler externals from",
+        default=None,
+    )
+    args = parser.parse_args()
+
+    # Load existing content if the file already exists (merge mode).
+    existing = {}
+    if os.path.isfile(args.output):
+        with open(args.output) as fid:
+            existing = yaml.safe_load(fid) or {}
+
+    compiler_packages = build_compiler_packages(args.compilers)
+
+    # Pull in any system compiler externals (e.g. system gcc) that carry
+    # extra_attributes.compilers but are not in the spack store.
+    if args.system_packages and os.path.isfile(args.system_packages):
+        system_externals = load_system_compiler_externals(args.system_packages)
+        for pkg_name, pkg_data in system_externals.items():
+            if pkg_name not in compiler_packages:
+                compiler_packages[pkg_name] = pkg_data
+
+    # Merge: compiler entries overwrite any existing entry for the same package name.
+    merged = existing.copy()
+    merged.setdefault("packages", {}).update(compiler_packages)
+
+    with open(args.output, "w") as fid:
+        yaml.dump(merged, fid, default_flow_style=False)
+
+    print(f"  compiler-config: wrote {args.output}", file=sys.stderr)
+
+
+main()

stackinator/etc/envvars.py

@@ -428,6 +428,12 @@ def read_activation_script(filename: str, env: Optional[EnvVarSet] = None) -> En
     if env is None:
         env = EnvVarSet()
 
+    # any prefix_paths that match entries in _IGNORE_PREFIX_PATHS will be dropped.
+    # e.g. /user/bin is in PATH in activate.sh scripts because it is in
+    # the environment used to build the uenv - but we want to avoid it so that
+    # it does not shadow PATH values set when more than 1 uenv is mounted.
+    _IGNORE_PREFIX_PATHS = {"PATH": ("/usr/bin", "/usr/local/bin", "/bin")}
+
     with open(filename) as fid:
         for line in fid:
             ls = line.strip().rstrip(";")
@@ -459,8 +465,12 @@ def read_activation_script(filename: str, env: Optional[EnvVarSet] = None) -> En
                 rhs = fields[1].lstrip("'").rstrip("'")
                 if name in list_variables:
                     fields = [f for f in rhs.split(":") if len(f.strip()) > 0]
-                    # look for $name as one of the fields (only works for append or prepend)
 
+                    # filter prefixes
+                    ignored_fields = _IGNORE_PREFIX_PATHS.get(name, ())
+                    fields = [f for f in fields if f not in ignored_fields]
+
+                    # look for $name as one of the fields (only works for append or prepend)
                     if len(fields) == 0:
                         env.set_list(name, fields, EnvVarOp.SET)
                     elif fields[0] == f"${name}":
@@ -476,11 +486,6 @@ def read_activation_script(filename: str, env: Optional[EnvVarSet] = None) -> En
 
 
 def view_impl(args):
-    print(
-        f"parsing view {args.root}\n  compilers {args.compilers}\n  prefix_paths '{args.prefix_paths}'\n  \
-        build_path '{args.build_path}'"
-    )
-
     if not os.path.isdir(args.root):
         print(f"error - environment root path {args.root} does not exist")
         exit(1)
@@ -503,6 +508,11 @@ def view_impl(args):
     # remove all prefix path variable values that point to a location inside the build path.
     envvars.remove_root(args.build_path)
 
+    # Canonical symlink names for gcc role keys. For other compiler families
+    # (nvhpc, llvm, etc.) the binary names are already canonical, so we fall
+    # back to os.path.basename which gives the right name (nvc, clang, etc.).
+    _GCC_ROLE_CANONICAL = {"c": "gcc", "cxx": "g++", "fortran": "gfortran"}
+
     if args.compilers is not None:
         if not os.path.isfile(args.compilers):
             print(f"error - compiler yaml file {args.compilers} does not exist")
@@ -511,28 +521,36 @@ def view_impl(args):
         with open(args.compilers, "r") as file:
             data = yaml.safe_load(file)
 
-        compilers = []
-        for p in data["packages"].values():
-            for e in p["externals"]:
-                if "extra_attributes" in e:
-                    c = e["extra_attributes"]["compilers"]
-                    if c is not None:
-                        compilers.append(c)
-
-        for c in compilers:
-            source_paths = list(set([os.path.abspath(v) for _, v in c.items() if v is not None]))
-            target_paths = [os.path.join(bin_path, os.path.basename(f)) for f in source_paths]
-            for src, dst in zip(source_paths, target_paths):
-                print(f"creating compiler symlink: {src} -> {dst}")
-                if os.path.exists(dst):
-                    print(f"  first removing {dst}")
-                    os.remove(dst)
-                os.symlink(src, dst)
+        # Restrict the symlinked compilers to those wired to this view's
+        # environment (environments.yaml: compiler). When None, link them all.
+        allowed = None
+        if args.compiler_names:
+            allowed = {n for n in args.compiler_names.split(",") if n}
 
-    if args.prefix_paths:
-        # get the root path of the env
-        print(f"prefix_paths: searching in {root_path}")
+        for pkg_name, pkg_data in data["packages"].items():
+            if allowed is not None and pkg_name not in allowed:
+                continue
+            for e in pkg_data["externals"]:
+                if "extra_attributes" not in e:
+                    continue
+                c = e["extra_attributes"].get("compilers")
+                if not c:
+                    continue
+                print(f"compiler symlinks: creating for {e.get('prefix', pkg_name)}")
+                for role, path in c.items():
+                    if path is None:
+                        continue
+                    src = os.path.abspath(path)
+                    if pkg_name == "gcc":
+                        link_name = _GCC_ROLE_CANONICAL.get(role, os.path.basename(src))
+                    else:
+                        link_name = os.path.basename(src)
+                    dst = os.path.join(bin_path, link_name)
+                    if os.path.exists(dst):
+                        os.remove(dst)
+                    os.symlink(src, dst)
 
+    if args.prefix_paths:
         for p in args.prefix_paths.split(","):
             name, value = p.split("=")
             paths = []
@@ -541,10 +559,6 @@ def view_impl(args):
                 if os.path.isdir(test_path):
                     paths.append(test_path)
 
-            print(f"{name}:")
-            for p in paths:
-                print(f"  {p}")
-
             if len(paths) > 0:
                 if name in envvars.lists:
                     ld_paths = envvars.lists[name].paths
@@ -678,6 +692,13 @@ def meta_impl(args):
     )
     # only add compilers if this argument is passed
     view_parser.add_argument("--compilers", help="path of the packages.yaml file", type=str, default=None)
+    view_parser.add_argument(
+        "--compiler-names",
+        help="comma-separated compiler package names to symlink into the view; "
+        "restricts --compilers to the compilers wired to this environment",
+        type=str,
+        default=None,
+    )
 
     uenv_parser = subparsers.add_parser(
         "uenv",
@@ -704,8 +725,6 @@ def meta_impl(args):
     args = parser.parse_args()
 
     if args.command == "uenv":
-        print("!!! running meta")
         meta_impl(args)
     elif args.command == "view":
-        print("!!! running view")
         view_impl(args)

stackinator/main.py

@@ -136,7 +136,7 @@ def main():
         root_logger.info("\nConfiguration finished, run the following to build the environment:\n")
         root_logger.info(f"cd {builder.path}")
         root_logger.info(
-            "env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin HOME=$HOME make store.squashfs -j32"
+            "env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin HOME=$HOME make store.squashfs NJOBS=32"
         )
         root_logger.info(f"see logfile for more information {logfile}")
         return 0