From 6b740f55ce43b57dfea66a4328bb283db049170b Mon Sep 17 00:00:00 2001 From: TheAbider <51920546+TheAbider@users.noreply.github.com> Date: Wed, 1 Jul 2026 04:11:12 -0700 Subject: [PATCH] Failover Clustering safety: quorum-preserving drain/evict, CSV data-loss guard, witness validation (v1.121.6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deep audit of the clustering module found call sites that failed to reuse the module's own correct safety patterns. - Dashboard node-drain now checks for another Up node + quorum before Suspend-ClusterNode -Drain (mirrors Suspend-ClusterNodeForMaintenance), instead of a bare confirm that could take the cluster offline (51-ClusterDashboard). - Node-eviction undo uses size-aware post-eviction quorum math instead of a hardcoded '-le 2' that let 4+ node clusters lose quorum (27-FailoverClustering). - CSV removal aborts when the mount point can't be resolved, instead of running a VM-dependency scan that matches nothing and falsely reports 'no VMs' — which could destroy live-VM storage. - Disk witness must be Online; file-share witness gets a Test-Path reachability pre-flight + a cluster-name-object ACL reminder. - Show-CSVHealth uses Get-ClusterSharedVolumeState.FileSystemRedirectedIOReason instead of the always-true SharedVolumeInfo.FaultState comparison (no more false 'redirected I/O' warnings on healthy CSVs). - Cluster validation report is saved to a known path and its location printed. - Tests: new Run-Tests Section 193. Version bump to 1.121.6. --- Changelog.md | 13 +++++++ Header.ps1 | 2 +- Modules/00-Initialization.ps1 | 2 +- Modules/27-FailoverClustering.ps1 | 56 +++++++++++++++++++++++++++---- Modules/51-ClusterDashboard.ps1 | 40 ++++++++++++++++++++-- README.md | 2 +- RackStack.ps1 | 2 +- RackStack.psd1 | 2 +- Tests/Run-Tests.ps1 | 41 +++++++++++++++++++++- 9 files changed, 145 insertions(+), 15 deletions(-) diff --git a/Changelog.md b/Changelog.md index 8446bda..67c0d2b 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,18 @@ # Changelog +## v1.121.6 + +Failover Clustering safety — fixes found by a deep audit of the clustering module. + +- **Draining a node from the dashboard now protects quorum.** It refuses to drain if no other node is up to receive the roles, and warns (requiring you to type CONTINUE) if draining would drop the cluster below quorum — instead of a single yes/no that could take the whole cluster offline. +- **Node eviction won't silently break quorum on larger clusters.** The safety check was hardcoded for small clusters; it now calculates quorum for the actual cluster size, so it can't be bypassed on 4+ node clusters. +- **Removing a Cluster Shared Volume can't destroy live-VM storage on false information.** If the tool can't determine the CSV's mount point (e.g. it's offline/degraded), it now refuses the removal rather than reporting "no VMs" from a check that never actually ran. +- **Quorum witness validation.** A disk witness must be Online before it can be selected; a file-share witness path is checked for reachability up front, with a reminder that the cluster account needs Full Control on the share. +- **Correct CSV redirected-I/O detection.** The dashboard no longer shows a false "REDIRECTED I/O ACTIVE" warning on healthy volumes (it was reading the wrong property). +- **Cluster validation report location is now shown** so you can actually find the report. + +No module or CLI action changes (81 modules, 201 actions). + ## v1.121.5 Reliability sweep — fixes found by auditing every menu for the same class of issues you ran into. diff --git a/Header.ps1 b/Header.ps1 index 8bc19a1..0a1228e 100644 --- a/Header.ps1 +++ b/Header.ps1 @@ -30,7 +30,7 @@ 7h3 4b1d3r .VERSION - 1.121.5 + 1.121.6 .LAST UPDATED 07/01/2026 diff --git a/Modules/00-Initialization.ps1 b/Modules/00-Initialization.ps1 index 2dbb156..8c6d3e2 100644 --- a/Modules/00-Initialization.ps1 +++ b/Modules/00-Initialization.ps1 @@ -233,7 +233,7 @@ if (-not $PSCommandPath -and $script:ScriptPath) { if (-not $script:ModuleRoot -and $script:ScriptPath) { $script:ModuleRoot = [System.IO.Path]::GetDirectoryName($script:ScriptPath) } -$script:ScriptVersion = "1.121.5" +$script:ScriptVersion = "1.121.6" $script:ScriptStartTime = Get-Date # Post-update cleanup: UpdateSelf / Rollback leave a `.pending-delete` sibling next to RackStack.exe. diff --git a/Modules/27-FailoverClustering.ps1 b/Modules/27-FailoverClustering.ps1 index 77752e1..3474fa5 100644 --- a/Modules/27-FailoverClustering.ps1 +++ b/Modules/27-FailoverClustering.ps1 @@ -298,8 +298,11 @@ function New-ClusterWizard { Write-OutputColor "" -color "Info" Write-OutputColor " Running cluster validation (this may take several minutes)..." -color "Info" try { - Test-Cluster -Node $nodes -ReportName "ClusterValidation_$(Get-Date -Format 'yyyyMMdd_HHmmss')" - Write-OutputColor " Validation complete. Check the report for any issues." -color "Success" + # Save the validation report to a known location and tell the operator where it is (a + # bare -ReportName drops the .htm in an unstated default dir). Mirrors Test-ClusterValidation. + $valReportPath = Join-Path $script:TempPath "ClusterValidation_$(Get-Date -Format 'yyyyMMdd_HHmmss')" + Test-Cluster -Node $nodes -ReportName $valReportPath | Out-Null + Write-OutputColor " Validation complete. Report saved to: $valReportPath.htm" -color "Success" Write-OutputColor "" -color "Info" if (-not (Confirm-UserAction -Message "Proceed with cluster creation?")) { return @@ -387,9 +390,18 @@ function Add-NodeToCluster { return } $upNodes = @($liveNodes | Where-Object { $_.State -eq 'Up' }) - if ($upNodes.Count -le 2) { - Write-OutputColor " REFUSING to evict: only $($upNodes.Count) node(s) currently Up." -color "Error" - Write-OutputColor " Evicting '$Node' would leave the cluster without quorum." -color "Error" + # Size-aware quorum check. Eviction REMOVES a node, so compute quorum against the + # post-eviction node count. The old '-le 2' test only protected 2-3 node clusters — on a + # 5-node cluster with 3 Up it passed, but evicting one leaves 4 nodes / 2 Up while quorum + # needs floor(4/2)+1 = 3, silently losing quorum. + $totalAfter = $liveNodes.Count - 1 + $nodeWasUp = @($upNodes | Where-Object { $_.Name -eq $Node }).Count -gt 0 + $upAfter = if ($nodeWasUp) { $upNodes.Count - 1 } else { $upNodes.Count } + $quorumAfter = [math]::Floor($totalAfter / 2) + 1 + if ($totalAfter -lt 1 -or $upAfter -lt $quorumAfter) { + Write-OutputColor " REFUSING to evict '$Node'." -color "Error" + Write-OutputColor " After eviction the cluster would have $upAfter/$totalAfter Up node(s), but quorum requires $quorumAfter." -color "Error" + Write-OutputColor " This would leave the cluster without quorum." -color "Error" Write-OutputColor " If this is intentional, evict manually via Failover Cluster Manager." -color "Warning" return } @@ -579,6 +591,17 @@ function Edit-ClusterSharedVolume { $csvMountRoot = $info.FriendlyVolumeName.TrimEnd('\') } } catch { } + # If we can't resolve the CSV mount point, the VM-dependency scan below would + # match nothing and falsely report "no VMs" — the operator would then confirm a + # removal on false information and could destroy live-VM storage. Refuse instead. + if (-not $csvMountRoot) { + Write-OutputColor "" -color "Error" + Write-OutputColor " Could not determine the mount point of CSV '$csvName'." -color "Error" + Write-OutputColor " Cannot verify whether any VMs depend on it, so removal is refused for safety." -color "Error" + Write-OutputColor " (Usually means the CSV is offline/degraded — check its health first.)" -color "Warning" + Write-PressEnter + return + } $dependentVMs = @() try { $clusteredVMs = @(Get-ClusterGroup -ErrorAction SilentlyContinue | Where-Object { $_.GroupType -eq 'VirtualMachine' }) @@ -972,7 +995,8 @@ function Set-ClusterQuorumConfig { Write-OutputColor "" -color "Info" $idx = 1 foreach ($disk in $disks) { - Write-OutputColor " [$idx] $($disk.Name)" -color "Info" + $diskStateColor = if ($disk.State -eq 'Online') { "Info" } else { "Warning" } + Write-OutputColor " [$idx] $($disk.Name) [$($disk.State)]" -color $diskStateColor $idx++ } Write-OutputColor "" -color "Info" @@ -982,6 +1006,15 @@ function Set-ClusterQuorumConfig { if ($diskChoice -match '^\d+$') { $selIdx = [int]$diskChoice - 1 if ($selIdx -ge 0 -and $selIdx -lt $disks.Count) { + # A disk witness that is Offline/Failed still counts as a vote — if it's not + # actually available, the cluster loses quorum when that vote is needed. Refuse + # to nominate a non-Online disk as the witness. + if ($disks[$selIdx].State -ne "Online") { + Write-OutputColor " Disk '$($disks[$selIdx].Name)' is not Online (State: $($disks[$selIdx].State))." -color "Error" + Write-OutputColor " An Offline/Failed disk cannot serve as the quorum witness — bring it Online first." -color "Warning" + Write-PressEnter + return + } try { Set-ClusterQuorum -NodeAndDiskMajority $disks[$selIdx].Name -ErrorAction Stop Write-OutputColor " Quorum set to Node and Disk Majority." -color "Success" @@ -1009,6 +1042,17 @@ function Set-ClusterQuorumConfig { break } if ($sharePath) { + # Pre-flight: a UNC that merely LOOKS valid isn't enough. The classic file-share- + # witness failure is a reachable share that doesn't grant the cluster name object + # (CNO) permission — the witness vote then fails at the critical moment. Verify + # reachability and remind about the ACL before committing. + if (-not (Test-Path -LiteralPath $sharePath)) { + Write-OutputColor " Share '$sharePath' is not reachable from this node." -color "Error" + Write-OutputColor " Create/share it and confirm network access, then retry." -color "Warning" + break + } + Write-OutputColor " Reminder: the cluster name object (CNO, e.g. CLUSTERNAME`$) needs Full Control" -color "Warning" + Write-OutputColor " on this share (both the SMB share and NTFS ACLs), or the witness vote will fail." -color "Warning" try { Set-ClusterQuorum -NodeAndFileShareMajority $sharePath -ErrorAction Stop Write-OutputColor " Quorum set to Node and File Share Majority." -color "Success" diff --git a/Modules/51-ClusterDashboard.ps1 b/Modules/51-ClusterDashboard.ps1 index f596c79..c5af8f7 100644 --- a/Modules/51-ClusterDashboard.ps1 +++ b/Modules/51-ClusterDashboard.ps1 @@ -196,6 +196,34 @@ function Start-ClusterNodeDrain { $selectedNode = $nodeMap[$choice] + # Safety: refuse to drain if no OTHER Up node can host the migrated roles, and require a typed + # CONTINUE if draining would drop the cluster below quorum — a bare yes/no confirm is not enough + # for a quorum-affecting operation. ($nodes above is Up-only, so re-query ALL nodes for the + # quorum math.) Mirrors Suspend-ClusterNodeForMaintenance in 27-FailoverClustering.ps1. + $clusterNodes = @(Get-ClusterNode -ErrorAction SilentlyContinue) + $otherUp = @($clusterNodes | Where-Object { $_.State -eq 'Up' -and $_.Name -ne $selectedNode }) + if ($otherUp.Count -eq 0) { + Write-OutputColor "" -color "Error" + Write-OutputColor " REFUSING to drain '$selectedNode': no other Up node is available to host its roles." -color "Error" + Write-OutputColor " Draining would leave roles unhosted and take the cluster offline." -color "Error" + Write-PressEnter + return + } + if ($clusterNodes.Count -gt 0) { + $quorumThreshold = [math]::Floor($clusterNodes.Count / 2) + 1 + if ($otherUp.Count -lt $quorumThreshold) { + Write-OutputColor "" -color "Warning" + Write-OutputColor " CAUTION: after draining, only $($otherUp.Count)/$($clusterNodes.Count) nodes will be Up" -color "Warning" + Write-OutputColor " (quorum requires $quorumThreshold). The cluster may halt if any other node fails." -color "Warning" + Write-OutputColor " Type CONTINUE to proceed:" -color "Warning" + $drainConfirm = Read-Host + if ($drainConfirm -ne 'CONTINUE') { + Write-OutputColor " Cancelled." -color "Info" + return + } + } + } + Write-OutputColor "" -color "Info" Write-OutputColor " Draining node: $selectedNode" -color "Warning" Write-OutputColor " This will migrate all VMs to other nodes and pause the node." -color "Info" @@ -315,7 +343,13 @@ function Show-CSVHealth { foreach ($csv in $csvs) { $partition = $csv.SharedVolumeInfo.Partition - $redirected = $csv.SharedVolumeInfo.FaultState + # Correct redirected-I/O detection: SharedVolumeInfo.FaultState reports fault status (e.g. + # "NoFaults"), never a redirection string, so the old '-ne "NoRedirectedAccess"' test warned + # on EVERY healthy CSV. Use Get-ClusterSharedVolumeState.FileSystemRedirectedIOReason — the + # API this module already uses correctly elsewhere. + $redirectedState = @($csv | Get-ClusterSharedVolumeState -ErrorAction SilentlyContinue) | + Where-Object { $_.FileSystemRedirectedIOReason -and $_.FileSystemRedirectedIOReason -ne 'NotRedirected' } | + Select-Object -First 1 if (-not $partition) { $lineStr = " $($csv.Name) - Partition info unavailable" if ($lineStr.Length -gt 69) { $lineStr = $lineStr.Substring(0, 69) + "..." } @@ -348,8 +382,8 @@ function Show-CSVHealth { Write-OutputColor " │$(" Space: ${usedGB}GB used / ${totalGB}GB total (${usedPct}% used)".PadRight(72))│" -color $spaceColor Write-OutputColor " │$(" Free: ${freeGB}GB".PadRight(72))│" -color $spaceColor - # Redirected I/O warning - if ($redirected -ne "NoRedirectedAccess") { + # Redirected I/O warning (only when genuinely redirected) + if ($redirectedState) { Write-OutputColor " │$(" ⚠ REDIRECTED I/O ACTIVE - Performance degraded!".PadRight(72))│" -color "Error" } diff --git a/README.md b/README.md index a11e347..f2595db 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ OpenSSF Best Practices codecov PSScriptAnalyzer 0 errors - 5269 structural tests + 5279 structural tests Pester 312 tests SLSA Level 3

diff --git a/RackStack.ps1 b/RackStack.ps1 index 70985ab..f54ad2e 100644 --- a/RackStack.ps1 +++ b/RackStack.ps1 @@ -13,7 +13,7 @@ Environment-specific settings are configured via defaults.json. .VERSION - 1.121.5 + 1.121.6 .NOTES - Requires Windows Server 2012 R2 or later (or Windows 10/11 for testing) - Must be run as Administrator diff --git a/RackStack.psd1 b/RackStack.psd1 index b3dbf13..69dcf31 100644 --- a/RackStack.psd1 +++ b/RackStack.psd1 @@ -1,6 +1,6 @@ @{ RootModule = 'RackStack.psm1' - ModuleVersion = '1.121.5' + ModuleVersion = '1.121.6' GUID = 'c19b8e71-4a35-4f2b-9d06-8a24f7bc0e91' Author = 'TheAbider' CompanyName = 'TheAbider' diff --git a/Tests/Run-Tests.ps1 b/Tests/Run-Tests.ps1 index 9a26a1f..935fa66 100644 --- a/Tests/Run-Tests.ps1 +++ b/Tests/Run-Tests.ps1 @@ -1,6 +1,6 @@ <# .SYNOPSIS - Automated Test Runner for RackStack v1.121.5 + Automated Test Runner for RackStack v1.121.6 .DESCRIPTION Comprehensive non-interactive test suite covering: @@ -9740,6 +9740,45 @@ catch { Write-TestResult "Reliability Sweep Tests" $false $_.Exception.Message } +# ============================================================================ +# SECTION 193: FAILOVER CLUSTERING SAFETY (v1.121.6) +# ============================================================================ +# Guards the cluster-safety fixes: quorum-preserving drain/evict, CSV-removal data-loss guard, +# disk/file-share witness validation, correct CSV redirected-I/O detection, report path. +Write-SectionHeader "SECTION 193: FAILOVER CLUSTERING SAFETY" + +try { + $fcR = Get-Content "$modulesPath\27-FailoverClustering.ps1" -Raw + $cdR = Get-Content "$modulesPath\51-ClusterDashboard.ps1" -Raw + + # Dashboard node-drain now checks other-Up-nodes + quorum before Suspend-ClusterNode -Drain. + Write-TestResult "51-ClusterDashboard: drain refuses when no other Up node" ($cdR -match 'REFUSING to drain[\s\S]{0,1500}Suspend-ClusterNode') + Write-TestResult "51-ClusterDashboard: drain has quorum-threshold gate" ($cdR -match 'quorumThreshold = \[math\]::Floor[\s\S]{0,1200}Suspend-ClusterNode') + + # Node-eviction undo uses size-aware post-eviction quorum math (no hardcoded -le 2). + Write-TestResult "27-FC: eviction undo is size-aware (no hardcoded -le 2)" ($fcR -match '\$quorumAfter = \[math\]::Floor\(\$totalAfter / 2\) \+ 1' -and -not ($fcR -match 'if \(\$upNodes\.Count -le 2\)')) + + # CSV removal aborts when the mount point can't be resolved (no false 'no VMs'). + Write-TestResult "27-FC: CSV removal aborts on unresolved mount point" ($fcR -match 'if \(-not \$csvMountRoot\)[\s\S]{0,300}refused for safety') + + # Disk witness must be Online before being nominated. + Write-TestResult "27-FC: disk witness requires Online state" ($fcR -match 'State -ne "Online"[\s\S]{0,300}Set-ClusterQuorum -NodeAndDiskMajority' -or $fcR -match 'State -ne "Online"[\s\S]{0,300}cannot serve as the quorum witness') + + # File-share witness has a reachability pre-flight + CNO ACL reminder. + Write-TestResult "27-FC: file-share witness pre-flights reachability" ($fcR -match 'Test-Path -LiteralPath \$sharePath[\s\S]{0,700}Set-ClusterQuorum -NodeAndFileShareMajority') + Write-TestResult "27-FC: file-share witness reminds about CNO ACL" ($fcR -match 'cluster name object \(CNO') + + # CSV redirected-I/O uses the correct API (not the always-true FaultState comparison). + Write-TestResult "51-ClusterDashboard: CSV redirect uses Get-ClusterSharedVolumeState" ($cdR -match 'Get-ClusterSharedVolumeState[\s\S]{0,200}FileSystemRedirectedIOReason') + Write-TestResult "51-ClusterDashboard: no bogus FaultState redirect check" (-not ($cdR -match '\$redirected -ne "NoRedirectedAccess"')) + + # Cluster validation report is saved to a known path and its location shown. + Write-TestResult "27-FC: wizard validation report path is shown" ($fcR -match 'Test-Cluster -Node \$nodes -ReportName \$valReportPath[\s\S]{0,120}Report saved to') +} +catch { + Write-TestResult "Failover Clustering Safety Tests" $false $_.Exception.Message +} + # ============================================================================ # SECTION 174: DOCUMENTATION FRESHNESS (counts must match the codebase) # ============================================================================