Implement NX-OS reprovisioning via NX-API

hack/nxapi/main.go

@@ -66,7 +66,7 @@ func main() {
 	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt /* == syscall.SIGINT */, syscall.SIGTERM)
 	defer cancel()
 
-	c, err := nxapi.NewClient(conn, 0)
+	c, err := nxapi.NewClient(conn)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "error: %v\n", err)
 		return

internal/controller/core/device_controller.go

@@ -427,83 +427,97 @@ func (r *DeviceReconciler) reconcileMaintenance(ctx context.Context, obj *v1alph
 	if !ok {
 		return nil
 	}
-	delete(obj.Annotations, v1alpha1.DeviceMaintenanceAnnotation)
 
 	switch action {
-	case v1alpha1.DeviceMaintenanceReboot:
-		prov, ok := r.Provider().(provider.MaintenanceProvider)
-		if !ok {
-			r.Recorder.Eventf(obj, nil, "Warning", "MaintenanceUnsupported", "Maintenance", "Provider does not support maintenance operation: %s", action)
-			return nil
-		}
-		// Reboot triggers a device restart. The device remains in its current phase
-		// and will resume normal operation after the reboot completes.
-		r.Recorder.Eventf(obj, nil, "Normal", "RebootRequested", "Maintenance", "Device reboot has been requested")
-		if err := prov.Reboot(ctx, conn); err != nil {
-			conditions.Set(obj, metav1.Condition{
-				Type:    v1alpha1.ReadyCondition,
-				Status:  metav1.ConditionFalse,
-				Reason:  v1alpha1.MaintenanceFailedReason,
-				Message: fmt.Sprintf("Failed to reboot device: %v", err),
-			})
-			r.Recorder.Eventf(obj, nil, "Warning", "RebootFailed", "Maintenance", "Device reboot has failed: %v", err)
-			return fmt.Errorf("failed to reboot device: %w", err)
-		}
-
-	case v1alpha1.DeviceMaintenanceFactoryReset:
-		prov, ok := r.Provider().(provider.MaintenanceProvider)
-		if !ok {
-			r.Recorder.Eventf(obj, nil, "Warning", "MaintenanceUnsupported", "Maintenance", "Provider does not support maintenance operation: %s", action)
-			return nil
-		}
-		// FactoryReset erases all device configuration and returns it to its original state.
-		// After completion, the device phase is reset to Pending to restart the lifecycle.
-		r.Recorder.Eventf(obj, nil, "Normal", "FactoryResetRequested", "Maintenance", "Device factory reset has been requested")
-		if err := prov.FactoryReset(ctx, conn); err != nil {
-			conditions.Set(obj, metav1.Condition{
-				Type:    v1alpha1.ReadyCondition,
-				Status:  metav1.ConditionFalse,
-				Reason:  v1alpha1.MaintenanceFailedReason,
-				Message: fmt.Sprintf("Failed to factory reset device: %v", err),
-			})
-			r.Recorder.Eventf(obj, nil, "Warning", "FactoryResetFailed", "Maintenance", "Device factory reset has failed: %v", err)
-			return fmt.Errorf("failed to reset device to factory defaults: %w", err)
-		}
-		obj.Status.Phase = v1alpha1.DevicePhasePending
-
-	case v1alpha1.DeviceMaintenanceReprovision:
-		prov, ok := r.Provider().(provider.ProvisioningProvider)
-		if !ok {
-			r.Recorder.Eventf(obj, nil, "Warning", "MaintenanceUnsupported", "Maintenance", "Provider does not support provisioning operation: %s", action)
-			return nil
-		}
-		// Reprovision prepares the device for re-provisioning without a full factory reset.
-		// The provider initiates the provisioning process, then the phase is reset to Pending.
-		r.Recorder.Eventf(obj, nil, "Normal", "ReprovisionRequested", "Maintenance", "Device reprovisioning has been requested")
-		if err := prov.Reprovision(ctx, conn); err != nil {
-			conditions.Set(obj, metav1.Condition{
-				Type:    v1alpha1.ReadyCondition,
-				Status:  metav1.ConditionFalse,
-				Reason:  v1alpha1.MaintenanceFailedReason,
-				Message: fmt.Sprintf("Failed to prepare device for reprovisioning: %v", err),
-			})
-			r.Recorder.Eventf(obj, nil, "Warning", "ReprovisionFailed", "Maintenance", "Device reprovisioning preparation has failed: %v", err)
-			return fmt.Errorf("failed to prepare device for reprovisioning: %w", err)
-		}
-		obj.Status.Phase = v1alpha1.DevicePhasePending
-
 	case v1alpha1.DeviceMaintenanceResetPhase:
 		// Reset phase is a soft reset that only changes the device phase to Pending without
 		// performing any device-side operations. This is useful for recovering from terminal
 		// states (e.g., Failed) after manual intervention.
 		r.Recorder.Eventf(obj, nil, "Normal", "PhaseReset", "Maintenance", "Device phase has been reset to Pending")
 		obj.Status.Phase = v1alpha1.DevicePhasePending
 
+	case v1alpha1.DeviceMaintenanceReboot,
+		v1alpha1.DeviceMaintenanceFactoryReset,
+		v1alpha1.DeviceMaintenanceReprovision:
+
+		prov := r.Provider()
+		if err := prov.Connect(ctx, conn); err != nil {
+			return fmt.Errorf("failed to connect to device: %w", err)
+		}
+		defer prov.Disconnect(ctx, conn) //nolint:errcheck
+
+		switch action {
+		case v1alpha1.DeviceMaintenanceReboot:
+			mp, ok := prov.(provider.MaintenanceProvider)
+			if !ok {
+				r.Recorder.Eventf(obj, nil, "Warning", "MaintenanceUnsupported", "Maintenance", "Provider does not support maintenance operation: %s", action)
+				return nil
+			}
+			// Reboot triggers a device restart. The device remains in its current phase
+			// and will resume normal operation after the reboot completes.
+			r.Recorder.Eventf(obj, nil, "Normal", "RebootRequested", "Maintenance", "Device reboot has been requested")
+			if err := mp.Reboot(ctx, conn); err != nil {
+				conditions.Set(obj, metav1.Condition{
+					Type:    v1alpha1.ReadyCondition,
+					Status:  metav1.ConditionFalse,
+					Reason:  v1alpha1.MaintenanceFailedReason,
+					Message: fmt.Sprintf("Failed to reboot device: %v", err),
+				})
+				r.Recorder.Eventf(obj, nil, "Warning", "RebootFailed", "Maintenance", "Device reboot has failed: %v", err)
+				return fmt.Errorf("failed to reboot device: %w", err)
+			}
+
+		case v1alpha1.DeviceMaintenanceFactoryReset:
+			mp, ok := prov.(provider.MaintenanceProvider)
+			if !ok {
+				r.Recorder.Eventf(obj, nil, "Warning", "MaintenanceUnsupported", "Maintenance", "Provider does not support maintenance operation: %s", action)
+				return nil
+			}
+			// FactoryReset erases all device configuration and returns it to its original state.
+			// After completion, the device phase is reset to Pending to restart the lifecycle.
+			r.Recorder.Eventf(obj, nil, "Normal", "FactoryResetRequested", "Maintenance", "Device factory reset has been requested")
+			if err := mp.FactoryReset(ctx, conn); err != nil {
+				conditions.Set(obj, metav1.Condition{
+					Type:    v1alpha1.ReadyCondition,
+					Status:  metav1.ConditionFalse,
+					Reason:  v1alpha1.MaintenanceFailedReason,
+					Message: fmt.Sprintf("Failed to factory reset device: %v", err),
+				})
+				r.Recorder.Eventf(obj, nil, "Warning", "FactoryResetFailed", "Maintenance", "Device factory reset has failed: %v", err)
+				return fmt.Errorf("failed to reset device to factory defaults: %w", err)
+			}
+			obj.Status.Phase = v1alpha1.DevicePhasePending
+
+		case v1alpha1.DeviceMaintenanceReprovision:
+			pp, ok := prov.(provider.ProvisioningProvider)
+			if !ok {
+				r.Recorder.Eventf(obj, nil, "Warning", "MaintenanceUnsupported", "Maintenance", "Provider does not support provisioning operation: %s", action)
+				return nil
+			}
+			// Reprovision prepares the device for re-provisioning without a full factory reset.
+			// The provider initiates the provisioning process, then the phase is reset to Pending.
+			r.Recorder.Eventf(obj, nil, "Normal", "ReprovisionRequested", "Maintenance", "Device reprovisioning has been requested")
+			if err := pp.Reprovision(ctx, conn); err != nil {
+				conditions.Set(obj, metav1.Condition{
+					Type:    v1alpha1.ReadyCondition,
+					Status:  metav1.ConditionFalse,
+					Reason:  v1alpha1.MaintenanceFailedReason,
+					Message: fmt.Sprintf("Failed to prepare device for reprovisioning: %v", err),
+				})
+				r.Recorder.Eventf(obj, nil, "Warning", "ReprovisionFailed", "Maintenance", "Device reprovisioning preparation has failed: %v", err)
+				return fmt.Errorf("failed to prepare device for reprovisioning: %w", err)
+			}
+			obj.Status.Phase = v1alpha1.DevicePhasePending
+		}
+
 	default:
 		r.Recorder.Eventf(obj, nil, "Warning", "UnknownMaintenanceAction", "Maintenance", "Unknown maintenance action: %s", action)
-		return fmt.Errorf("unknown maintenance action: %s", action)
+		return reconcile.TerminalError(fmt.Errorf("unknown maintenance action: %s", action))
 	}
 
+	// Only remove the annotation after the operation succeeds so that
+	// failed actions are retried on the next reconciliation.
+	delete(obj.Annotations, v1alpha1.DeviceMaintenanceAnnotation)
 	return nil
 }
 

internal/provider/cisco/nxos/provider.go

@@ -76,13 +76,14 @@ type Provider struct {
 	nxapi  *nxapi.Client
 }
 
+// timeout is the default timeout for all HTTP/gRPC requests made by the provider.
+const timeout = 30 * time.Second
+
 func NewProvider() provider.Provider {
 	return &Provider{}
 }
 
 func (p *Provider) Connect(ctx context.Context, conn *deviceutil.Connection) (err error) {
-	// timeout is the default timeout for all HTTP/gRPC requests made by the provider.
-	const timeout = 30 * time.Second
 	p.conn, err = grpcext.NewClient(conn, grpcext.WithDefaultTimeout(timeout))
 	if err != nil {
 		return fmt.Errorf("failed to create grpc connection: %w", err)
@@ -98,7 +99,7 @@ func (p *Provider) Connect(ctx context.Context, conn *deviceutil.Connection) (er
 	// NXAPI only uses the address for URI construction.
 	c := *conn
 	c.Address = netip.MustParseAddrPort(conn.Address).Addr().String()
-	p.nxapi, err = nxapi.NewClient(&c, timeout)
+	p.nxapi, err = nxapi.NewClient(&c, nxapi.WithTimeout(timeout))
 	if err != nil {
 		return fmt.Errorf("failed to create nxapi client: %w", err)
 	}
@@ -147,23 +148,24 @@ func (p *Provider) FactoryReset(ctx context.Context, conn *deviceutil.Connection
 	return FactoryReset(ctx, p.conn)
 }
 
-func (p *Provider) Reprovision(ctx context.Context, conn *deviceutil.Connection) (reterr error) {
-	if err := p.Connect(ctx, conn); err != nil {
-		return err
+func (p *Provider) Reprovision(ctx context.Context, conn *deviceutil.Connection) error {
+	_, err := p.nxapi.Do(ctx, nxapi.NewRequest(
+		"boot poap enable",
+		"copy running-config startup-config",
+	).WithRollback(nxapi.Stop))
+	if err != nil {
+		return fmt.Errorf("failed to prepare device for reprovisioning: %w", err)
 	}
-	defer func() {
-		if err := p.Disconnect(ctx, conn); err != nil {
-			reterr = errors.Join(reterr, err)
-		}
-	}()
-	// This is currently defunct on NX-OS, as enabling POAP requires a `copy running-config startup-config` which we
-	// cannot issue via GNMI
-	// TODO add once NXAPI client is available
-	poap := BootPOAP("enable")
-	if err := p.client.Update(ctx, &poap); err != nil {
-		return err
+
+	// Reboot is issued as a separate request because it actually restarts
+	// the device. The connection will drop before a response is received,
+	// so transport errors are expected and tolerated.
+	_, err = p.nxapi.Do(ctx, nxapi.NewRequest("reload"))
+	if err != nil && !nxapi.IsTransportError(err) {
+		return fmt.Errorf("failed to reboot device: %w", err)
 	}
-	return Reboot(ctx, p.conn)
+
+	return nil
 }
 
 func (p *Provider) ListPorts(ctx context.Context) ([]provider.DevicePort, error) {

internal/provider/cisco/nxos/reprovision_test.go

@@ -0,0 +1,136 @@
+// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package nxos
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"testing"
+
+	"github.com/ironcore-dev/network-operator/internal/deviceutil"
+	"github.com/ironcore-dev/network-operator/internal/transport/nxapi"
+)
+
+func TestReprovision(t *testing.T) {
+	t.Run("success with connection drop on reload", func(t *testing.T) {
+		var requests [][]string
+		called := 0
+
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			var cmds []struct {
+				Params struct {
+					Cmd string `json:"cmd"`
+				} `json:"params"`
+			}
+			if err := json.NewDecoder(r.Body).Decode(&cmds); err != nil {
+				t.Fatalf("failed to decode request: %v", err)
+			}
+
+			batch := make([]string, len(cmds))
+			for i, c := range cmds {
+				batch[i] = c.Params.Cmd
+			}
+			requests = append(requests, batch)
+			called++
+
+			if called == 1 {
+				w.Header().Set("Content-Type", "application/json-rpc")
+				fmt.Fprint(w, `[
+					{"jsonrpc":"2.0","result":null,"id":1},
+					{"jsonrpc":"2.0","result":null,"id":2}
+				]`)
+				return
+			}
+
+			// Simulate device going down by closing connection abruptly.
+			hijacker, ok := w.(http.Hijacker)
+			if !ok {
+				t.Fatal("server does not support hijacking")
+			}
+			conn, _, err := hijacker.Hijack()
+			if err != nil {
+				t.Fatalf("hijack failed: %v", err)
+			}
+			conn.Close()
+		}))
+		defer srv.Close()
+
+		_, port, _ := net.SplitHostPort(srv.Listener.Addr().String()) //nolint:errcheck // httptest address is always host:port
+		conn := &deviceutil.Connection{Address: srv.Listener.Addr().String(), Username: "admin", Password: "secret"}
+		client, err := nxapi.NewClient(conn, nxapi.WithPort(port))
+		if err != nil {
+			t.Fatalf("failed to create nxapi client: %v", err)
+		}
+
+		p := &Provider{nxapi: client}
+		if err := p.Reprovision(t.Context(), conn); err != nil {
+			t.Fatalf("Reprovision returned unexpected error: %v", err)
+		}
+
+		if len(requests) != 2 {
+			t.Fatalf("expected 2 NXAPI requests, got %d", len(requests))
+		}
+		if !slices.Equal(requests[0], []string{"boot poap enable", "copy running-config startup-config"}) {
+			t.Errorf("prep batch = %v", requests[0])
+		}
+		if !slices.Equal(requests[1], []string{"reload"}) {
+			t.Errorf("reload request = %v", requests[1])
+		}
+	})
+
+	t.Run("prep batch RPC error fails", func(t *testing.T) {
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "application/json-rpc")
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprint(w, `[{"jsonrpc":"2.0","error":{"code":-32602,"message":"Invalid command"},"id":1}]`)
+		}))
+		defer srv.Close()
+
+		_, port, _ := net.SplitHostPort(srv.Listener.Addr().String()) //nolint:errcheck // httptest address is always host:port
+		conn := &deviceutil.Connection{Address: srv.Listener.Addr().String(), Username: "admin", Password: "secret"}
+		client, err := nxapi.NewClient(conn, nxapi.WithPort(port))
+		if err != nil {
+			t.Fatalf("failed to create nxapi client: %v", err)
+		}
+
+		p := &Provider{nxapi: client}
+		if err := p.Reprovision(t.Context(), conn); err == nil {
+			t.Fatal("expected error from prep batch, got nil")
+		}
+	})
+
+	t.Run("reload RPC error fails", func(t *testing.T) {
+		called := 0
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			called++
+			w.Header().Set("Content-Type", "application/json-rpc")
+			if called == 1 {
+				fmt.Fprint(w, `[
+					{"jsonrpc":"2.0","result":null,"id":1},
+					{"jsonrpc":"2.0","result":null,"id":2}
+				]`)
+				return
+			}
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprint(w, `[{"jsonrpc":"2.0","error":{"code":-32602,"message":"Permission denied"},"id":1}]`)
+		}))
+		defer srv.Close()
+
+		_, port, _ := net.SplitHostPort(srv.Listener.Addr().String()) //nolint:errcheck // httptest address is always host:port
+		conn := &deviceutil.Connection{Address: srv.Listener.Addr().String(), Username: "admin", Password: "secret"}
+		client, err := nxapi.NewClient(conn, nxapi.WithPort(port))
+		if err != nil {
+			t.Fatalf("failed to create nxapi client: %v", err)
+		}
+
+		p := &Provider{nxapi: client}
+		if err := p.Reprovision(t.Context(), conn); err == nil {
+			t.Fatal("expected error from reload RPC failure, got nil")
+		}
+	})
+}

internal/provider/cisco/nxos/system.go

@@ -68,14 +68,6 @@ func (*FirmwareVersion) XPath() string {
 	return "System/showversion-items/nxosVersion"
 }
 
-var _ gnmiext.DataElement = (*BootPOAP)(nil)
-
-type BootPOAP string
-
-func (*BootPOAP) XPath() string {
-	return "/System/boot-items/poap"
-}
-
 type BootTime UnixTime
 
 func (*BootTime) XPath() string {