Add optional mounts

src/deploy_tools/app_builder.py

@@ -57,6 +57,9 @@ def _create_apptainer_files(self, app: ApptainerApp, module: Module) -> None:
             entrypoint_file = entrypoints_folder / entrypoint.name
 
             mounts = ",".join(chain(global_options.mounts, options.mounts)).strip()
+            optional_mounts = ",".join(
+                chain(global_options.optional_mounts, options.optional_mounts)
+            ).strip()
             host_binaries = " ".join(
                 chain(global_options.host_binaries, options.host_binaries)
             ).strip()
@@ -71,6 +74,7 @@ def _create_apptainer_files(self, app: ApptainerApp, module: Module) -> None:
 
             params = {
                 "mounts": mounts,
+                "optional_mounts": optional_mounts,
                 "host_binaries": host_binaries,
                 "apptainer_args": apptainer_args,
                 "relative_sif_file": relative_sif_file,

src/deploy_tools/models/apptainer_app.py

@@ -1,6 +1,6 @@
 from typing import Annotated, Literal
 
-from pydantic import AnyUrl, Field, StringConstraints, UrlConstraints
+from pydantic import AnyUrl, Field, StringConstraints, UrlConstraints, model_validator
 
 from .app import ENTRYPOINT_NAME_REGEX
 from .parent import ParentModel
@@ -27,9 +27,20 @@ class EntrypointOptions(ParentModel):
     mounts: Annotated[
         list[MountPoint],
         Field(
-            description="A list of mount points to add to the container in the form of "
+            description="A list of mount points that will result in an error if their "
+            "host paths cannot be found. This takes the form of "
             "'host_path[:container_path[:opts]]' where opts (mount options) can be "
-            "'ro' or 'rw' and defaults to 'rw'"
+            "'ro' or 'rw' and defaults to 'rw'. "
+        ),
+    ] = []
+
+    optional_mounts: Annotated[
+        list[MountPoint],
+        Field(
+            description="A list of mount points that will not be mounted if their host "
+            "paths cannot be found (avoiding an error). This takes the form of "
+            "'host_path[:container_path[:opts]]' where opts (mount options) "
+            "can be 'ro' or 'rw' and defaults to 'rw'."
         ),
     ] = []
 
@@ -42,6 +53,17 @@ class EntrypointOptions(ParentModel):
         ),
     ] = []
 
+    @model_validator(mode="after")
+    def check_unique_mounts(self) -> "EntrypointOptions":
+        """Ensure that mounts and optional_mounts do not contain duplicates."""
+        duplicate_mounts = set(self.mounts).intersection(set(self.optional_mounts))
+        if duplicate_mounts:
+            raise ValueError(
+                f"Duplicate paths found in mounts and optional_mounts: "
+                f"{duplicate_mounts}"
+            )
+        return self
+
 
 class Entrypoint(ParentModel):
     """Represents an entrypoint to a command on the Apptainer image.
@@ -121,3 +143,14 @@ class ApptainerApp(ParentModel):
         EntrypointOptions,
         Field(description="Global options that apply to all Entrypoints"),
     ] = EntrypointOptions()
+
+    @model_validator(mode="after")
+    def prioritise_entrypoint_mounts(self) -> "ApptainerApp":
+        """Allow entrypoint options to override mounts defined in global_options"""
+        for entrypoint in self.entrypoints:
+            for mount in entrypoint.options.mounts + entrypoint.options.optional_mounts:
+                if mount in self.global_options.mounts:
+                    self.global_options.mounts.remove(mount)
+                if mount in self.global_options.optional_mounts:
+                    self.global_options.optional_mounts.remove(mount)
+        return self

src/deploy_tools/models/schemas/deployment.json

@@ -28,6 +28,7 @@
             "apptainer_args": "",
             "command_args": "",
             "mounts": [],
+            "optional_mounts": [],
             "host_binaries": []
           },
           "description": "Global options that apply to all Entrypoints"
@@ -157,6 +158,7 @@
             "apptainer_args": "",
             "command_args": "",
             "mounts": [],
+            "optional_mounts": [],
             "host_binaries": []
           },
           "description": "Options to apply for this entrypoint"
@@ -186,13 +188,22 @@
         },
         "mounts": {
           "default": [],
-          "description": "A list of mount points to add to the container in the form of 'host_path[:container_path[:opts]]' where opts (mount options) can be 'ro' or 'rw' and defaults to 'rw'",
+          "description": "A list of mount points that will result in an error if their host paths cannot be found. This takes the form of 'host_path[:container_path[:opts]]' where opts (mount options) can be 'ro' or 'rw' and defaults to 'rw'. ",
           "items": {
             "$ref": "#/$defs/MountPoint"
           },
           "title": "Mounts",
           "type": "array"
         },
+        "optional_mounts": {
+          "default": [],
+          "description": "A list of mount points that will not be mounted if their host paths cannot be found (avoiding an error). This takes the form of 'host_path[:container_path[:opts]]' where opts (mount options) can be 'ro' or 'rw' and defaults to 'rw'.",
+          "items": {
+            "$ref": "#/$defs/MountPoint"
+          },
+          "title": "Optional Mounts",
+          "type": "array"
+        },
         "host_binaries": {
           "default": [],
           "description": "A list of host binaries to mount into the container. These are discovered on the host using the current PATH and are mounted into the container at /usr/bin/[binary_name]",

src/deploy_tools/models/schemas/module.json

@@ -28,6 +28,7 @@
             "apptainer_args": "",
             "command_args": "",
             "mounts": [],
+            "optional_mounts": [],
             "host_binaries": []
           },
           "description": "Global options that apply to all Entrypoints"
@@ -138,6 +139,7 @@
             "apptainer_args": "",
             "command_args": "",
             "mounts": [],
+            "optional_mounts": [],
             "host_binaries": []
           },
           "description": "Options to apply for this entrypoint"
@@ -167,13 +169,22 @@
         },
         "mounts": {
           "default": [],
-          "description": "A list of mount points to add to the container in the form of 'host_path[:container_path[:opts]]' where opts (mount options) can be 'ro' or 'rw' and defaults to 'rw'",
+          "description": "A list of mount points that will result in an error if their host paths cannot be found. This takes the form of 'host_path[:container_path[:opts]]' where opts (mount options) can be 'ro' or 'rw' and defaults to 'rw'. ",
           "items": {
             "$ref": "#/$defs/MountPoint"
           },
           "title": "Mounts",
           "type": "array"
         },
+        "optional_mounts": {
+          "default": [],
+          "description": "A list of mount points that will not be mounted if their host paths cannot be found (avoiding an error). This takes the form of 'host_path[:container_path[:opts]]' where opts (mount options) can be 'ro' or 'rw' and defaults to 'rw'.",
+          "items": {
+            "$ref": "#/$defs/MountPoint"
+          },
+          "title": "Optional Mounts",
+          "type": "array"
+        },
         "host_binaries": {
           "default": [],
           "description": "A list of host binaries to mount into the container. These are discovered on the host using the current PATH and are mounted into the container at /usr/bin/[binary_name]",

src/deploy_tools/models/schemas/release.json

@@ -28,6 +28,7 @@
             "apptainer_args": "",
             "command_args": "",
             "mounts": [],
+            "optional_mounts": [],
             "host_binaries": []
           },
           "description": "Global options that apply to all Entrypoints"
@@ -138,6 +139,7 @@
             "apptainer_args": "",
             "command_args": "",
             "mounts": [],
+            "optional_mounts": [],
             "host_binaries": []
           },
           "description": "Options to apply for this entrypoint"
@@ -167,13 +169,22 @@
         },
         "mounts": {
           "default": [],
-          "description": "A list of mount points to add to the container in the form of 'host_path[:container_path[:opts]]' where opts (mount options) can be 'ro' or 'rw' and defaults to 'rw'",
+          "description": "A list of mount points that will result in an error if their host paths cannot be found. This takes the form of 'host_path[:container_path[:opts]]' where opts (mount options) can be 'ro' or 'rw' and defaults to 'rw'. ",
           "items": {
             "$ref": "#/$defs/MountPoint"
           },
           "title": "Mounts",
           "type": "array"
         },
+        "optional_mounts": {
+          "default": [],
+          "description": "A list of mount points that will not be mounted if their host paths cannot be found (avoiding an error). This takes the form of 'host_path[:container_path[:opts]]' where opts (mount options) can be 'ro' or 'rw' and defaults to 'rw'.",
+          "items": {
+            "$ref": "#/$defs/MountPoint"
+          },
+          "title": "Optional Mounts",
+          "type": "array"
+        },
         "host_binaries": {
           "default": [],
           "description": "A list of host binaries to mount into the container. These are discovered on the host using the current PATH and are mounted into the container at /usr/bin/[binary_name]",

src/deploy_tools/templates/apptainer_entrypoint

@@ -8,6 +8,7 @@ set -e
 
 # Mounts for container
 mounts="{{ mounts }}"
+optional_mounts="{{ optional_mounts }}"
 # Additional arguments for apptainer
 apptainer_args="{{ apptainer_args }}"
 # Sif file path
@@ -17,6 +18,25 @@ command="{{ command }}"
 # Options and arguments to pass to command
 command_args="{{ command_args }}"
 
+{% if optional_mounts|length %}
+# Filter out any invalid mounts and warn the user
+IFS="," read -ra optional_mount_paths <<< "$optional_mounts"
+
+validated_mounts=()
+
+for mount_path in "${optional_mount_paths[@]}"; do
+    host_path="${mount_path%%:*}"
+
+    if [[ -d "$host_path" || -f "$host_path" ]]; then
+        validated_mounts+=("$mount_path")
+    else
+        echo "WARNING: Unable to mount $host_path: path does not exist" >&2
+    fi
+done
+
+mounts="${mounts},$(IFS=','; echo "${validated_mounts[*]}")"
+{% endif -%}
+
 # Raise an error if sif file does not exist
 if [[ ! -f ${sif_file} ]]; then
     echo "ERROR: sif file ${sif_file} does not exist" 1>&2

tests/samples/01-initial/deploy-tools-output/deployment.yaml

@@ -17,11 +17,13 @@ releases:
               command_args: Hello
               host_binaries: []
               mounts: []
+              optional_mounts: []
           global_options:
             apptainer_args: ''
             command_args: ''
             host_binaries: []
             mounts: []
+            optional_mounts: []
         - app_type: shell
           name: test-echo-module-var
           script:

tests/samples/01-initial/deploy-tools-output/modules/apps/0.1/entrypoints/cowsay-hello

@@ -8,6 +8,7 @@ set -e
 
 # Mounts for container
 mounts=""
+optional_mounts=""
 # Additional arguments for apptainer
 apptainer_args=""
 # Sif file path
@@ -17,6 +18,23 @@ command="cowsay"
 # Options and arguments to pass to command
 command_args="Hello"
 
+# Filter out any invalid mounts and warn the user
+IFS="," read -ra optional_mount_paths <<< "$optional_mounts"
+
+validated_mounts=()
+
+for mount_path in "${optional_mount_paths[@]}"; do
+    host_path="${mount_path%%:*}"
+
+    if [[ -d "$host_path" || -f "$host_path" ]]; then
+        validated_mounts+=("$mount_path")
+    else
+        echo "WARNING: Unable to mount $host_path: path does not exist" >&2
+    fi
+done
+
+mounts="${mounts},$(IFS=','; echo "${validated_mounts[*]}")"
+
 # Raise an error if sif file does not exist
 if [[ ! -f ${sif_file} ]]; then
     echo "ERROR: sif file ${sif_file} does not exist" 1>&2

tests/samples/01-initial/deploy-tools-output/modules/apps/0.1/module.yaml

@@ -12,11 +12,13 @@ applications:
       command_args: Hello
       host_binaries: []
       mounts: []
+      optional_mounts: []
   global_options:
     apptainer_args: ''
     command_args: ''
     host_binaries: []
     mounts: []
+    optional_mounts: []
 - app_type: shell
   name: test-echo-module-var
   script:

tests/samples/02-added/deploy-tools-output/deployment.yaml

@@ -17,11 +17,13 @@ releases:
               command_args: Hello
               host_binaries: []
               mounts: []
+              optional_mounts: []
           global_options:
             apptainer_args: ''
             command_args: ''
             host_binaries: []
             mounts: []
+            optional_mounts: []
         - app_type: shell
           name: test-echo-module-var
           script:

tests/samples/02-added/deploy-tools-output/modules/apps/0.1/entrypoints/cowsay-hello

@@ -8,6 +8,7 @@ set -e
 
 # Mounts for container
 mounts=""
+optional_mounts=""
 # Additional arguments for apptainer
 apptainer_args=""
 # Sif file path
@@ -17,6 +18,23 @@ command="cowsay"
 # Options and arguments to pass to command
 command_args="Hello"
 
+# Filter out any invalid mounts and warn the user
+IFS="," read -ra optional_mount_paths <<< "$optional_mounts"
+
+validated_mounts=()
+
+for mount_path in "${optional_mount_paths[@]}"; do
+    host_path="${mount_path%%:*}"
+
+    if [[ -d "$host_path" || -f "$host_path" ]]; then
+        validated_mounts+=("$mount_path")
+    else
+        echo "WARNING: Unable to mount $host_path: path does not exist" >&2
+    fi
+done
+
+mounts="${mounts},$(IFS=','; echo "${validated_mounts[*]}")"
+
 # Raise an error if sif file does not exist
 if [[ ! -f ${sif_file} ]]; then
     echo "ERROR: sif file ${sif_file} does not exist" 1>&2

tests/samples/02-added/deploy-tools-output/modules/apps/0.1/module.yaml

@@ -12,11 +12,13 @@ applications:
       command_args: Hello
       host_binaries: []
       mounts: []
+      optional_mounts: []
   global_options:
     apptainer_args: ''
     command_args: ''
     host_binaries: []
     mounts: []
+    optional_mounts: []
 - app_type: shell
   name: test-echo-module-var
   script:

tests/samples/03-updated/deploy-tools-output/deployment.yaml

@@ -17,11 +17,13 @@ releases:
               command_args: Hello
               host_binaries: []
               mounts: []
+              optional_mounts: []
           global_options:
             apptainer_args: ''
             command_args: ''
             host_binaries: []
             mounts: []
+            optional_mounts: []
         - app_type: shell
           name: test-echo-module-var
           script: