Enhance DICOM processing and add HPC deployment support

.gitignore

@@ -21,3 +21,7 @@ import os.py
 reset_02.sh
 *.log
 *.json
+.aider*
+
+# Singularity images (large, site-specific)
+*.sif

code/preprocessing/00_preprocess.sh

@@ -1,14 +1,93 @@
-#!/bin/bash
-
-python /FL_system/code/preprocessing/01_scanDicom.py
-echo "01 Completed" # Used by script.js to check status of the process
-python /FL_system/code/preprocessing/02_parseDicom.py
-echo "02 Completed" # Used by script.js to check status of the process
-python /FL_system/code/preprocessing/03_saveNifti.py
-echo "03 Completed" # Used by script.js to check status of the process
-python /FL_system/code/preprocessing/04_saveRAS.py
-echo "04 Completed" # Used by script.js to check status of the process
-python /FL_system/code/preprocessing/05_alignScans.py
-echo "05 Completed" # Used by script.js to check status of the process
-python /FL_system/code/preprocessing/06_genInputs.py
-echo "06 Completed" # Used by script.js to check status of the process
+# =============================================================================
+# 00_preprocess.sh — MRI preprocessing pipeline orchestrator
+#
+# Usage:
+#   bash 00_preprocess.sh                          (runs all 6 steps)
+#   bash 00_preprocess.sh --start-step 3            (steps 03-06 only)
+#   bash 00_preprocess.sh --stop-step 4             (steps 01-04 only)
+#   bash 00_preprocess.sh --steps 1,3,5             (only listed steps)
+#   bash 00_preprocess.sh --scan-dir /path/raw      (override scan path)
+#   bash 00_preprocess.sh --save-dir /path/output   (override save path)
+# =============================================================================
+
+set -euo pipefail
+
+SCAN_DIR=""
+SAVE_DIR=""
+START_STEP=1
+STOP_STEP=6
+STEPS_FILTER=""
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --scan-dir)   SCAN_DIR="$2"; shift 2 ;;
+    --save-dir)   SAVE_DIR="$2"; shift 2 ;;
+    --start-step) START_STEP="$2"; shift 2 ;;
+    --stop-step)  STOP_STEP="$2"; shift 2 ;;
+    --steps)      STEPS_FILTER="$2"; shift 2 ;;
+    *)            shift ;;
+  esac
+done
+
+should_run() {
+  if [ "$1" -lt "$START_STEP" ] || [ "$1" -gt "$STOP_STEP" ]; then
+    return 1
+  fi
+  if [ -n "$STEPS_FILTER" ] && [[ ! ",$STEPS_FILTER," == *",$1,"* ]]; then
+    return 1
+  fi
+  return 0
+}
+
+# Step 01
+if should_run 1; then
+  STEP01_ARGS=()
+  [ -n "$SCAN_DIR" ] && STEP01_ARGS+=("--scan-dir" "$SCAN_DIR")
+  [ -n "$SAVE_DIR" ] && STEP01_ARGS+=("--save-dir" "$SAVE_DIR")
+  python /FL_system/code/preprocessing/01_scanDicom.py "${STEP01_ARGS[@]}"
+else
+  echo "Skipping step 01"
+fi
+echo "01 Completed"
+
+# Step 02
+if should_run 2; then
+  python /FL_system/code/preprocessing/02_parseDicom.py
+else
+  echo "Skipping step 02"
+fi
+echo "02 Completed"
+
+# Step 03
+if should_run 3; then
+  python /FL_system/code/preprocessing/03_saveNifti.py
+else
+  echo "Skipping step 03"
+fi
+echo "03 Completed"
+
+# Step 04
+if should_run 4; then
+  python /FL_system/code/preprocessing/04_saveRAS.py
+else
+  echo "Skipping step 04"
+fi
+echo "04 Completed"
+
+# Step 05
+if should_run 5; then
+  python /FL_system/code/preprocessing/05_alignScans.py
+else
+  echo "Skipping step 05"
+fi
+echo "05 Completed"
+
+# Step 06
+if should_run 6; then
+  python /FL_system/code/preprocessing/06_genInputs.py
+else
+  echo "Skipping step 06"
+fi
+echo "06 Completed"
+
+echo "Pipeline complete."

code/scripts/install_niftyreg.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# =============================================================================
+# Install niftyreg v2.0.0 from source
+# =============================================================================
+#
+# niftyreg is a C++/CUDA registration toolkit required for step 05 (alignScans).
+# It is NOT available via conda-forge, so this helper builds it locally.
+#
+# Usage:
+#   bash scripts/install_niftyreg.sh [install_prefix]
+#
+# Default install prefix: ~/mri_niftyreg
+#
+# Requirements on host:
+#   - gcc, g++, cmake (conda provides these or use system modules)
+#   - CUDA toolkit (optional, for GPU-accelerated registration)
+#
+# =============================================================================
+
+set -euo pipefail
+
+PREFIX="${1:-${HOME}/mri_niftyreg}"
+BUILD_DIR="/tmp/niftyreg_build_${RANDOM}"
+
+echo "┌─────────────────────────────────────────────────────┐"
+echo "│ NiftyReg v2.0.0 Installer                            │"
+echo "└─────────────────────────────────────────────────────┘"
+echo ""
+
+# ── Check build requirements ──────────────────────────────────────
+if ! command -v cmake &>/dev/null; then
+  echo "ERROR: cmake not found. Install via: conda install cmake"
+  exit 1
+fi
+
+if ! command -v make &>/dev/null; then
+  echo "WARNING: make not found, trying ninja..."
+  if ! command -v ninja &>/dev/null; then
+    exit 1
+  fi
+fi
+
+# ── Check for CUDA (optional, for GPU mode) ───────────────────────
+CUDA_FOUND=false
+if command -v nvcc &>/dev/null; then
+  CUDA_VERSION=$(nvcc --version | grep -i "release" | awk -F',' '{gsub(/ /, "", $3); print $3}')
+  echo "✓ CUDA ${CUDA_VERSION} found — building with GPU acceleration"
+  CUDA_FOUND=true
+else
+  echo "⚠ CUDA not found — building without GPU acceleration (CPU-only mode)"
+fi
+
+# ── Create build directory ────────────────────────────────────────
+rm -rf "${BUILD_DIR}"
+mkdir -p "${BUILD_DIR}"
+mkdir -p "${PREFIX}"
+
+echo "→ Cloning niftyreg v2.0.0..."
+git clone --branch v2.0.0 https://github.com/KCL-BMEIS/niftyreg.git "${BUILD_DIR}/niftyreg-git"
+
+echo "→ Configuring build (CUDA=${CUDA_FOUND})..."
+cd "${BUILD_DIR}/niftyreg-git"
+mkdir -p build
+cd build
+
+CMAKE_CUDA_FLAG="-DBUILD_CUDA=ON"
+if [[ "${CUDA_FOUND}" = false ]]; then
+  CMAKE_CUDA_FLAG="-DBUILD_CUDA=OFF"
+fi
+
+cmake .. \
+  ${CMAKE_CUDA_FLAG} \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_INSTALL_PREFIX="${PREFIX}"
+
+echo "→ Compiling (this takes ~10 minutes)..."
+make install
+
+echo "→ Cleaning up build files..."
+rm -rf "${BUILD_DIR}"
+
+echo ""
+echo "═════════════════════════════════════════════"
+echo "✓ NiftyReg installed to: ${PREFIX}"
+echo ""
+echo "Add to PATH before running pipeline:"
+echo "  export PATH=${PREFIX}/bin:\${PATH}"
+echo ""
+echo "Then run:"
+echo "  ./run_pipeline_conda.sh"
+echo "═════════════════════════════════════════════" 

control_system/dockerfile

@@ -5,7 +5,7 @@
 #     -it -v ${PROJECT_DIRECTORY_PATH}:/FL_system -v ${DATA_DIRECTORY_PATH}:/FL_system/data/raw \
 #     mri_preprocessing bash
 
-FROM nvidia/cuda:12.2.2-base-ubuntu22.04
+FROM nvidia/cuda:12.2.2-base-ubuntu22.04 AS base
 
 RUN apt-get update && \
     apt-get install -y python3-pip python3-dev && \

control_system/mri_preprocessing.singularity.def

@@ -0,0 +1,27 @@
+Bootstrap: docker-source
+Source: nvidia/cuda:12.2.2-base-ubuntu2204
+
+%labels
+  AUTHOR TheParraLab
+  VERSION  v0.9.0
+  DESCRIPTION "MRI preprocessing pipeline (DICOM→NIfTI conversion + alignment via niftyreg)"
+  NVIDIA_CUDA_VERSION 12.2.2_ubuntu2204
+
+%post
+    apt-get update && \
+    apt-get install -y python3-pip python3-dev dcm2niix git cmake g++ ca-certificates curl gnupg && \
+    if [ ! -L /usr/bin/python ] && [ ! -e /usr/bin/python ]; then ln -s /usr/bin/python3 /usr/bin/python; fi && \
+    python3 -m pip install --upgrade pip && \
+    git clone --branch v2.0.0 https://github.com/KCL-BMEIS/niftyreg.git niftyreg-git && \
+    mkdir niftyreg-git/build && cd niftyreg-git/build && \
+    cmake .. -DBUILD_CUDA=ON && make install && cd ~ && rm -rf ~/niftyreg-git && \
+    apt-get remove --purge -y git cmake g++ ca-certificates curl gnupg && \
+    if [ ! -e /var/cache/apt/archives/lock ]; then rm -rf /var/lib/apt/lists/*; fi
+
+%post pip3 install pydicom numpy pandas nibabel scipy yappi --no-cache-dir
+
+%environment
+    export PATH=/usr/local/bin:$PATH  
+  PYTHONTYPE=python3  
+
+%start exec "$@"

environment.yml

@@ -0,0 +1,16 @@
+name: mri_preproc
+channels:
+  - conda-forge
+  - defaults
+
+dependencies:
+  - python=3
+  - pydicom
+  - numpy
+  - pandas
+  - nibabel
+  - scipy
+  - yappi
+  - dcm2niix
+  - pytest>=7
+  - pytest-cov>=3

run_pipeline_conda.sh

@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+# =============================================================================
+# MRI Preprocessing — Conda-only pipeline runner
+# =============================================================================
+#
+# For HPC sites that don't support Docker or Singularity. Requires conda/mamba.
+#
+# Usage:
+#   1) Clone this repo onto HPC
+#   2) Run `./setup_conda.sh`      (one-time: creates conda env + installs niftyreg)
+#      OR skip if niftyreg already available as an HPC module
+#   3) Run `./start_control.sh`    (same script as today — auto-detects conda fallback)
+#   4) Run `bash code/preprocessing/00_preprocess.sh`
+#
+# ── NiftyReg availability ─────────────────────────────────────────────────
+#
+# niftyreg is NOT bundled via conda (CUDA build). Options:
+#   Option A — Use an existing HPC module (preferred)
+#     module load niftyreg
+#   Option B — Build manually (requires gcc, cmake on site)
+#     ./scripts/install_niftyreg.sh
+#   Option C — Copy a pre-built `.sif` image from a Docker build and run via Singularity
+#
+# =============================================================================
+
+set -euo pipefail
+
+# ── Detect script root ─────────────────────────────────────────────
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ENV_YML="${SCRIPT_DIR}/environment.yml"
+ENV_NAME="mri_preproc"
+NIFTYREG_MODULE_AVAILABLE=false
+NIFTYREG_SYSTEM_INSTALL=false
+
+# ── Prompt paths ───────────────────────────────────────────────────
+echo "┌─────────────────────────────────────────────────────┐"
+echo "│ MRI Preprocessing — Conda Pipeline                   │"
+echo "└─────────────────────────────────────────────────────┘"
+echo ""
+
+echo "Please enter the raw DICOM data path:"
+read -r DATA_DIRECTORY_PATH
+
+echo "Please enter the NIfTI output path:"  
+read -r NIFTI_DIRECTORY_PATH
+
+PROJECT_DIRECTORY_PATH="${SCRIPT_DIR}"
+
+# ── Export env vars for all pipeline scripts ──────────────────────
+export PROJECT_DIRECTORY_PATH
+export DATA_DIRECTORY_PATH
+export NIFTI_DIRECTORY_PATH
+
+# ── Check for existing conda env ──────────────────────────────────
+CONDAPATH=""
+if command -v mamba &>/dev/null; then
+  CONDAPATH=$(mamba info --base 2>/dev/null) || true
+  CMD=mamba
+elif command -v conda &>/dev/null; then
+  CONDAPATH=$(conda info --base 2>/dev/null) || true
+  CMD=conda
+fi
+
+if [[ -z "${CONDAPATH}" ]]; then
+  echo ""
+  echo "ERROR: Neither conda nor mamba found. Install one of:"
+  echo "  Conda:  https://docs.conda.io/en/latest/miniconda.html"
+  echo "  Mamba:  https://mamba.readthedocs.io/en/latest/installation/mamba-installation.html"
+  echo ""
+  echo "Then re-run this script."
+  exit 1
+fi
+
+if [[ -d "${CONDAPATH}/envs/${ENV_NAME}" ]]; then
+  echo "Environment ${ENV_NAME} already exists. Activating..."
+else
+  echo ""
+  echo "Installing conda environment ${ENV_NAME}..."
+  ${CMD} env create -f "${ENV_YML}" --yes
+fi
+
+# ── Activate env ──────────────────────────────────────────────────
+if ${CMD} env list | grep -q "^${ENV_NAME}"; then
+  eval "$(${CMD} shell.bash hook)"
+  ${CMD} activate "${ENV_NAME}"
+  echo "→ ${ENV_NAME} activated successfully."
+else
+  echo "ERROR: Could not activate ${ENV_NAME} — did the install succeed?"
+  exit 1
+fi
+
+# ── Check for niftyreg ───────────────────────────────────────────
+if module load niftyreg 2>/dev/null; then
+  NIFTYREG_MODULE_AVAILABLE=true
+  echo "→ Found niftyreg via system module."
+fi
+
+if ! command -v reg_f3d &>/dev/null; then
+  echo ""
+  echo "WARNING: reg_f3d not found in PATH. niftyreg required for step 05."
+  echo ""
+  echo "Install options:"
+  echo "  1) module load niftyreg                        ← preferred if available"
+  echo "  2) ${SCRIPT_DIR}/scripts/install_niftyreg.sh   ← build from source"
+  echo ""
+  echo "Exiting — please install niftyreg and re-run."
+  exit 1
+fi
+
+# ── Resolve paths into container-equivalent dirs ─────────────────
+# All pipeline scripts expect paths under /FL_system/
+# We bind them directly since we're running natively (no container)
+
+# ── Verify dependencies ──────────────────────────────────────────
+echo ""
+echo "✓ dcm2niix:    $(command -v dcm2niix 2>/dev/null || echo 'NOT FOUND')"
+echo "✓ reg_f3d:     $(command -v reg_f3d 2>/dev/null || echo 'NOT FOUND')"
+echo "✓ python:      $(python --version 2>&1)"
+echo "✓ pydicom:     $(python -c 'import pydicom; print(pydicom.__version__)' 2>&1)"
+echo ""
+
+echo "──────────────────────────────────────────────────────────"
+echo "Pipeline ready. Run from project root:"
+echo "  bash code/preprocessing/00_preprocess.sh"
+echo "──────────────────────────────────────────────────────────"
+
+# Run the pipeline directly
+    cd "${PROJECT_DIRECTORY_PATH}"
+    bash code/preprocessing/00_preprocess.sh \
+      --scan-dir "${DATA_DIRECTORY_PATH}" \
+      --save-dir "${DATA_DIRECTORY_PATH}"