Failover Clustering safety: quorum guards + CSV data-loss guard + witness validation (v1.121.6)

Changelog.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## v1.121.6
+
+Failover Clustering safety — fixes found by a deep audit of the clustering module.
+
+- **Draining a node from the dashboard now protects quorum.** It refuses to drain if no other node is up to receive the roles, and warns (requiring you to type CONTINUE) if draining would drop the cluster below quorum — instead of a single yes/no that could take the whole cluster offline.
+- **Node eviction won't silently break quorum on larger clusters.** The safety check was hardcoded for small clusters; it now calculates quorum for the actual cluster size, so it can't be bypassed on 4+ node clusters.
+- **Removing a Cluster Shared Volume can't destroy live-VM storage on false information.** If the tool can't determine the CSV's mount point (e.g. it's offline/degraded), it now refuses the removal rather than reporting "no VMs" from a check that never actually ran.
+- **Quorum witness validation.** A disk witness must be Online before it can be selected; a file-share witness path is checked for reachability up front, with a reminder that the cluster account needs Full Control on the share.
+- **Correct CSV redirected-I/O detection.** The dashboard no longer shows a false "REDIRECTED I/O ACTIVE" warning on healthy volumes (it was reading the wrong property).
+- **Cluster validation report location is now shown** so you can actually find the report.
+
+No module or CLI action changes (81 modules, 201 actions).
+
 ## v1.121.5
 
 Reliability sweep — fixes found by auditing every menu for the same class of issues you ran into.

Header.ps1

@@ -30,7 +30,7 @@
     7h3 4b1d3r
 
 .VERSION
-    1.121.5
+    1.121.6
 .LAST UPDATED
     07/01/2026
 

Modules/00-Initialization.ps1

@@ -233,7 +233,7 @@ if (-not $PSCommandPath -and $script:ScriptPath) {
 if (-not $script:ModuleRoot -and $script:ScriptPath) {
     $script:ModuleRoot = [System.IO.Path]::GetDirectoryName($script:ScriptPath)
 }
-$script:ScriptVersion = "1.121.5"
+$script:ScriptVersion = "1.121.6"
 $script:ScriptStartTime = Get-Date
 
 # Post-update cleanup: UpdateSelf / Rollback leave a `.pending-delete` sibling next to RackStack.exe.

Modules/27-FailoverClustering.ps1

@@ -298,8 +298,11 @@ function New-ClusterWizard {
         Write-OutputColor "" -color "Info"
         Write-OutputColor "  Running cluster validation (this may take several minutes)..." -color "Info"
         try {
-            Test-Cluster -Node $nodes -ReportName "ClusterValidation_$(Get-Date -Format 'yyyyMMdd_HHmmss')"
-            Write-OutputColor "  Validation complete. Check the report for any issues." -color "Success"
+            # Save the validation report to a known location and tell the operator where it is (a
+            # bare -ReportName drops the .htm in an unstated default dir). Mirrors Test-ClusterValidation.
+            $valReportPath = Join-Path $script:TempPath "ClusterValidation_$(Get-Date -Format 'yyyyMMdd_HHmmss')"
+            Test-Cluster -Node $nodes -ReportName $valReportPath | Out-Null
+            Write-OutputColor "  Validation complete. Report saved to: $valReportPath.htm" -color "Success"
             Write-OutputColor "" -color "Info"
             if (-not (Confirm-UserAction -Message "Proceed with cluster creation?")) {
                 return
@@ -387,9 +390,18 @@ function Add-NodeToCluster {
                 return
             }
             $upNodes = @($liveNodes | Where-Object { $_.State -eq 'Up' })
-            if ($upNodes.Count -le 2) {
-                Write-OutputColor "  REFUSING to evict: only $($upNodes.Count) node(s) currently Up." -color "Error"
-                Write-OutputColor "  Evicting '$Node' would leave the cluster without quorum." -color "Error"
+            # Size-aware quorum check. Eviction REMOVES a node, so compute quorum against the
+            # post-eviction node count. The old '-le 2' test only protected 2-3 node clusters — on a
+            # 5-node cluster with 3 Up it passed, but evicting one leaves 4 nodes / 2 Up while quorum
+            # needs floor(4/2)+1 = 3, silently losing quorum.
+            $totalAfter = $liveNodes.Count - 1
+            $nodeWasUp = @($upNodes | Where-Object { $_.Name -eq $Node }).Count -gt 0
+            $upAfter = if ($nodeWasUp) { $upNodes.Count - 1 } else { $upNodes.Count }
+            $quorumAfter = [math]::Floor($totalAfter / 2) + 1
+            if ($totalAfter -lt 1 -or $upAfter -lt $quorumAfter) {
+                Write-OutputColor "  REFUSING to evict '$Node'." -color "Error"
+                Write-OutputColor "  After eviction the cluster would have $upAfter/$totalAfter Up node(s), but quorum requires $quorumAfter." -color "Error"
+                Write-OutputColor "  This would leave the cluster without quorum." -color "Error"
                 Write-OutputColor "  If this is intentional, evict manually via Failover Cluster Manager." -color "Warning"
                 return
             }
@@ -579,6 +591,17 @@ function Edit-ClusterSharedVolume {
                             $csvMountRoot = $info.FriendlyVolumeName.TrimEnd('\')
                         }
                     } catch { }
+                    # If we can't resolve the CSV mount point, the VM-dependency scan below would
+                    # match nothing and falsely report "no VMs" — the operator would then confirm a
+                    # removal on false information and could destroy live-VM storage. Refuse instead.
+                    if (-not $csvMountRoot) {
+                        Write-OutputColor "" -color "Error"
+                        Write-OutputColor "  Could not determine the mount point of CSV '$csvName'." -color "Error"
+                        Write-OutputColor "  Cannot verify whether any VMs depend on it, so removal is refused for safety." -color "Error"
+                        Write-OutputColor "  (Usually means the CSV is offline/degraded — check its health first.)" -color "Warning"
+                        Write-PressEnter
+                        return
+                    }
                     $dependentVMs = @()
                     try {
                         $clusteredVMs = @(Get-ClusterGroup -ErrorAction SilentlyContinue | Where-Object { $_.GroupType -eq 'VirtualMachine' })
@@ -972,7 +995,8 @@ function Set-ClusterQuorumConfig {
             Write-OutputColor "" -color "Info"
             $idx = 1
             foreach ($disk in $disks) {
-                Write-OutputColor "  [$idx] $($disk.Name)" -color "Info"
+                $diskStateColor = if ($disk.State -eq 'Online') { "Info" } else { "Warning" }
+                Write-OutputColor "  [$idx] $($disk.Name) [$($disk.State)]" -color $diskStateColor
                 $idx++
             }
             Write-OutputColor "" -color "Info"
@@ -982,6 +1006,15 @@ function Set-ClusterQuorumConfig {
             if ($diskChoice -match '^\d+$') {
                 $selIdx = [int]$diskChoice - 1
                 if ($selIdx -ge 0 -and $selIdx -lt $disks.Count) {
+                    # A disk witness that is Offline/Failed still counts as a vote — if it's not
+                    # actually available, the cluster loses quorum when that vote is needed. Refuse
+                    # to nominate a non-Online disk as the witness.
+                    if ($disks[$selIdx].State -ne "Online") {
+                        Write-OutputColor "  Disk '$($disks[$selIdx].Name)' is not Online (State: $($disks[$selIdx].State))." -color "Error"
+                        Write-OutputColor "  An Offline/Failed disk cannot serve as the quorum witness — bring it Online first." -color "Warning"
+                        Write-PressEnter
+                        return
+                    }
                     try {
                         Set-ClusterQuorum -NodeAndDiskMajority $disks[$selIdx].Name -ErrorAction Stop
                         Write-OutputColor "  Quorum set to Node and Disk Majority." -color "Success"
@@ -1009,6 +1042,17 @@ function Set-ClusterQuorumConfig {
                 break
             }
             if ($sharePath) {
+                # Pre-flight: a UNC that merely LOOKS valid isn't enough. The classic file-share-
+                # witness failure is a reachable share that doesn't grant the cluster name object
+                # (CNO) permission — the witness vote then fails at the critical moment. Verify
+                # reachability and remind about the ACL before committing.
+                if (-not (Test-Path -LiteralPath $sharePath)) {
+                    Write-OutputColor "  Share '$sharePath' is not reachable from this node." -color "Error"
+                    Write-OutputColor "  Create/share it and confirm network access, then retry." -color "Warning"
+                    break
+                }
+                Write-OutputColor "  Reminder: the cluster name object (CNO, e.g. CLUSTERNAME`$) needs Full Control" -color "Warning"
+                Write-OutputColor "  on this share (both the SMB share and NTFS ACLs), or the witness vote will fail." -color "Warning"
                 try {
                     Set-ClusterQuorum -NodeAndFileShareMajority $sharePath -ErrorAction Stop
                     Write-OutputColor "  Quorum set to Node and File Share Majority." -color "Success"

Modules/51-ClusterDashboard.ps1

@@ -196,6 +196,34 @@ function Start-ClusterNodeDrain {
 
     $selectedNode = $nodeMap[$choice]
 
+    # Safety: refuse to drain if no OTHER Up node can host the migrated roles, and require a typed
+    # CONTINUE if draining would drop the cluster below quorum — a bare yes/no confirm is not enough
+    # for a quorum-affecting operation. ($nodes above is Up-only, so re-query ALL nodes for the
+    # quorum math.) Mirrors Suspend-ClusterNodeForMaintenance in 27-FailoverClustering.ps1.
+    $clusterNodes = @(Get-ClusterNode -ErrorAction SilentlyContinue)
+    $otherUp = @($clusterNodes | Where-Object { $_.State -eq 'Up' -and $_.Name -ne $selectedNode })
+    if ($otherUp.Count -eq 0) {
+        Write-OutputColor "" -color "Error"
+        Write-OutputColor "  REFUSING to drain '$selectedNode': no other Up node is available to host its roles." -color "Error"
+        Write-OutputColor "  Draining would leave roles unhosted and take the cluster offline." -color "Error"
+        Write-PressEnter
+        return
+    }
+    if ($clusterNodes.Count -gt 0) {
+        $quorumThreshold = [math]::Floor($clusterNodes.Count / 2) + 1
+        if ($otherUp.Count -lt $quorumThreshold) {
+            Write-OutputColor "" -color "Warning"
+            Write-OutputColor "  CAUTION: after draining, only $($otherUp.Count)/$($clusterNodes.Count) nodes will be Up" -color "Warning"
+            Write-OutputColor "  (quorum requires $quorumThreshold). The cluster may halt if any other node fails." -color "Warning"
+            Write-OutputColor "  Type CONTINUE to proceed:" -color "Warning"
+            $drainConfirm = Read-Host
+            if ($drainConfirm -ne 'CONTINUE') {
+                Write-OutputColor "  Cancelled." -color "Info"
+                return
+            }
+        }
+    }
+
     Write-OutputColor "" -color "Info"
     Write-OutputColor "  Draining node: $selectedNode" -color "Warning"
     Write-OutputColor "  This will migrate all VMs to other nodes and pause the node." -color "Info"
@@ -315,7 +343,13 @@ function Show-CSVHealth {
 
     foreach ($csv in $csvs) {
         $partition = $csv.SharedVolumeInfo.Partition
-        $redirected = $csv.SharedVolumeInfo.FaultState
+        # Correct redirected-I/O detection: SharedVolumeInfo.FaultState reports fault status (e.g.
+        # "NoFaults"), never a redirection string, so the old '-ne "NoRedirectedAccess"' test warned
+        # on EVERY healthy CSV. Use Get-ClusterSharedVolumeState.FileSystemRedirectedIOReason — the
+        # API this module already uses correctly elsewhere.
+        $redirectedState = @($csv | Get-ClusterSharedVolumeState -ErrorAction SilentlyContinue) |
+            Where-Object { $_.FileSystemRedirectedIOReason -and $_.FileSystemRedirectedIOReason -ne 'NotRedirected' } |
+            Select-Object -First 1
         if (-not $partition) {
             $lineStr = "  $($csv.Name) - Partition info unavailable"
             if ($lineStr.Length -gt 69) { $lineStr = $lineStr.Substring(0, 69) + "..." }
@@ -348,8 +382,8 @@ function Show-CSVHealth {
         Write-OutputColor "  │$("  Space: ${usedGB}GB used / ${totalGB}GB total (${usedPct}% used)".PadRight(72))│" -color $spaceColor
         Write-OutputColor "  │$("  Free: ${freeGB}GB".PadRight(72))│" -color $spaceColor
 
-        # Redirected I/O warning
-        if ($redirected -ne "NoRedirectedAccess") {
+        # Redirected I/O warning (only when genuinely redirected)
+        if ($redirectedState) {
             Write-OutputColor "  │$("  ⚠ REDIRECTED I/O ACTIVE - Performance degraded!".PadRight(72))│" -color "Error"
         }
 

README.md

@@ -28,7 +28,7 @@
   <a href="https://www.bestpractices.dev/projects/12921"><img alt="OpenSSF Best Practices" src="https://www.bestpractices.dev/projects/12921/badge"></a>
   <a href="https://codecov.io/gh/TheAbider/RackStack"><img alt="codecov" src="https://codecov.io/gh/TheAbider/RackStack/branch/master/graph/badge.svg"></a>
   <img alt="PSScriptAnalyzer 0 errors" src="https://img.shields.io/badge/PSScriptAnalyzer-0%20errors-brightgreen">
-  <img alt="5269 structural tests" src="https://img.shields.io/badge/structural%20tests-5269-brightgreen">
+  <img alt="5279 structural tests" src="https://img.shields.io/badge/structural%20tests-5279-brightgreen">
   <img alt="Pester 312 tests" src="https://img.shields.io/badge/Pester-312%20tests-brightgreen">
   <img alt="SLSA Level 3" src="https://slsa.dev/images/gh-badge-level3.svg">
 </p>

RackStack.ps1

@@ -13,7 +13,7 @@
     Environment-specific settings are configured via defaults.json.
 
 .VERSION
-    1.121.5
+    1.121.6
 .NOTES
     - Requires Windows Server 2012 R2 or later (or Windows 10/11 for testing)
     - Must be run as Administrator

RackStack.psd1

@@ -1,6 +1,6 @@
 @{
     RootModule           = 'RackStack.psm1'
-    ModuleVersion        = '1.121.5'
+    ModuleVersion        = '1.121.6'
     GUID                 = 'c19b8e71-4a35-4f2b-9d06-8a24f7bc0e91'
     Author               = 'TheAbider'
     CompanyName          = 'TheAbider'

Tests/Run-Tests.ps1

@@ -1,6 +1,6 @@
 <#
 .SYNOPSIS
-    Automated Test Runner for RackStack v1.121.5
+    Automated Test Runner for RackStack v1.121.6
 
 .DESCRIPTION
     Comprehensive non-interactive test suite covering:
@@ -9740,6 +9740,45 @@ catch {
     Write-TestResult "Reliability Sweep Tests" $false $_.Exception.Message
 }
 
+# ============================================================================
+# SECTION 193: FAILOVER CLUSTERING SAFETY (v1.121.6)
+# ============================================================================
+# Guards the cluster-safety fixes: quorum-preserving drain/evict, CSV-removal data-loss guard,
+# disk/file-share witness validation, correct CSV redirected-I/O detection, report path.
+Write-SectionHeader "SECTION 193: FAILOVER CLUSTERING SAFETY"
+
+try {
+    $fcR = Get-Content "$modulesPath\27-FailoverClustering.ps1" -Raw
+    $cdR = Get-Content "$modulesPath\51-ClusterDashboard.ps1" -Raw
+
+    # Dashboard node-drain now checks other-Up-nodes + quorum before Suspend-ClusterNode -Drain.
+    Write-TestResult "51-ClusterDashboard: drain refuses when no other Up node" ($cdR -match 'REFUSING to drain[\s\S]{0,1500}Suspend-ClusterNode')
+    Write-TestResult "51-ClusterDashboard: drain has quorum-threshold gate" ($cdR -match 'quorumThreshold = \[math\]::Floor[\s\S]{0,1200}Suspend-ClusterNode')
+
+    # Node-eviction undo uses size-aware post-eviction quorum math (no hardcoded -le 2).
+    Write-TestResult "27-FC: eviction undo is size-aware (no hardcoded -le 2)" ($fcR -match '\$quorumAfter = \[math\]::Floor\(\$totalAfter / 2\) \+ 1' -and -not ($fcR -match 'if \(\$upNodes\.Count -le 2\)'))
+
+    # CSV removal aborts when the mount point can't be resolved (no false 'no VMs').
+    Write-TestResult "27-FC: CSV removal aborts on unresolved mount point" ($fcR -match 'if \(-not \$csvMountRoot\)[\s\S]{0,300}refused for safety')
+
+    # Disk witness must be Online before being nominated.
+    Write-TestResult "27-FC: disk witness requires Online state" ($fcR -match 'State -ne "Online"[\s\S]{0,300}Set-ClusterQuorum -NodeAndDiskMajority' -or $fcR -match 'State -ne "Online"[\s\S]{0,300}cannot serve as the quorum witness')
+
+    # File-share witness has a reachability pre-flight + CNO ACL reminder.
+    Write-TestResult "27-FC: file-share witness pre-flights reachability" ($fcR -match 'Test-Path -LiteralPath \$sharePath[\s\S]{0,700}Set-ClusterQuorum -NodeAndFileShareMajority')
+    Write-TestResult "27-FC: file-share witness reminds about CNO ACL" ($fcR -match 'cluster name object \(CNO')
+
+    # CSV redirected-I/O uses the correct API (not the always-true FaultState comparison).
+    Write-TestResult "51-ClusterDashboard: CSV redirect uses Get-ClusterSharedVolumeState" ($cdR -match 'Get-ClusterSharedVolumeState[\s\S]{0,200}FileSystemRedirectedIOReason')
+    Write-TestResult "51-ClusterDashboard: no bogus FaultState redirect check" (-not ($cdR -match '\$redirected -ne "NoRedirectedAccess"'))
+
+    # Cluster validation report is saved to a known path and its location shown.
+    Write-TestResult "27-FC: wizard validation report path is shown" ($fcR -match 'Test-Cluster -Node \$nodes -ReportName \$valReportPath[\s\S]{0,120}Report saved to')
+}
+catch {
+    Write-TestResult "Failover Clustering Safety Tests" $false $_.Exception.Message
+}
+
 # ============================================================================
 # SECTION 174: DOCUMENTATION FRESHNESS (counts must match the codebase)
 # ============================================================================