From 14979c1405032eb16a000a681525ecd767e387cd Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 7 Oct 2026 09:59:42 +0200 Subject: [PATCH 1/4] fix(rest): honour delete_namespaces on resource-definition and group modify Both modify bodies declare the props-modify envelope's three halves, and the merge applied only override_props and delete_props, so a modify carrying delete_namespaces answered 200 and changed nothing. Accepting a field and dropping it tells the operator work happened when it did not. The namespace match the volume-definition modify already had becomes deletePropNamespaces and serves all three: a namespace covers its own key and every key below it, never a key that merely starts like it. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- pkg/rest/props_delete_namespaces_test.go | 108 +++++++++++++++++++++++ pkg/rest/props_modify.go | 16 ++++ pkg/rest/resource_definitions.go | 6 ++ pkg/rest/resource_groups.go | 6 ++ pkg/rest/volume_definitions.go | 8 +- 5 files changed, 137 insertions(+), 7 deletions(-) create mode 100644 pkg/rest/props_delete_namespaces_test.go diff --git a/pkg/rest/props_delete_namespaces_test.go b/pkg/rest/props_delete_namespaces_test.go new file mode 100644 index 00000000..b0d03597 --- /dev/null +++ b/pkg/rest/props_delete_namespaces_test.go @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "encoding/json" + "net/http" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// The modify body declares delete_namespaces and the merge dropped it, so a +// modify carrying it answered 200 and changed nothing. +func TestRDModifyHonoursDeleteNamespaces(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "rd-ns", + Props: map[string]string{ + "DrbdOptions": "top", + "DrbdOptions/Net/protocol": "C", + "DrbdOptionsOther": "keep-me", + }, + }); err != nil { + t.Fatalf("seed RD: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(map[string]any{"delete_namespaces": []string{"DrbdOptions"}}) + + resp := httpPut(t, base+"/v1/resource-definitions/rd-ns", body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + + got, err := st.ResourceDefinitions().Get(ctx, "rd-ns") + if err != nil { + t.Fatalf("read the RD back: %v", err) + } + + for _, key := range []string{"DrbdOptions", "DrbdOptions/Net/protocol"} { + if _, present := got.Props[key]; present { + t.Errorf("prop %q survived delete_namespaces", key) + } + } + + // The neighbouring key that merely shares a prefix is not in the + // namespace and must stay. + if got.Props["DrbdOptionsOther"] != "keep-me" { + t.Errorf("DrbdOptionsOther = %q, want keep-me", got.Props["DrbdOptionsOther"]) + } +} + +// The resource-group modify declares delete_namespaces and merged only the +// other two halves of the envelope. +func TestRGModifyHonoursDeleteNamespaces(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + + if err := st.ResourceGroups().Create(ctx, &apiv1.ResourceGroup{ + Name: "rg-ns", + Props: map[string]string{ + "DrbdOptions": "top", + "DrbdOptions/Net/protocol": "C", + "DrbdOptionsOther": "keep-me", + }, + }); err != nil { + t.Fatalf("seed RG: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(map[string]any{"delete_namespaces": []string{"DrbdOptions"}}) + + resp := httpPut(t, base+"/v1/resource-groups/rg-ns", body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + + got, err := st.ResourceGroups().Get(ctx, "rg-ns") + if err != nil { + t.Fatalf("read the RG back: %v", err) + } + + for _, key := range []string{"DrbdOptions", "DrbdOptions/Net/protocol"} { + if _, present := got.Props[key]; present { + t.Errorf("prop %q survived delete_namespaces", key) + } + } + + if got.Props["DrbdOptionsOther"] != "keep-me" { + t.Errorf("DrbdOptionsOther = %q, want keep-me", got.Props["DrbdOptionsOther"]) + } +} diff --git a/pkg/rest/props_modify.go b/pkg/rest/props_modify.go index 50e3e995..014d33a1 100644 --- a/pkg/rest/props_modify.go +++ b/pkg/rest/props_modify.go @@ -65,3 +65,19 @@ func applyPropsModify(props, override map[string]string, del []string) map[strin return props } + +// deletePropNamespaces strips every property under the named namespaces, the +// `delete_namespaces` half of upstream's props-modify envelope. A namespace +// covers the key that spells it exactly and every key below it — `DrbdOptions` +// takes `DrbdOptions` and `DrbdOptions/Net/protocol` with it, and leaves +// `DrbdOptionsOther` alone, because the separator has to be there for a key to +// be inside the namespace rather than merely to start like it. +func deletePropNamespaces(props map[string]string, namespaces []string) { + for _, ns := range namespaces { + for key := range props { + if key == ns || (len(key) > len(ns) && key[:len(ns)] == ns && key[len(ns)] == '/') { + delete(props, key) + } + } + } +} diff --git a/pkg/rest/resource_definitions.go b/pkg/rest/resource_definitions.go index 12efbc4f..0adb8865 100644 --- a/pkg/rest/resource_definitions.go +++ b/pkg/rest/resource_definitions.go @@ -1062,6 +1062,12 @@ func (s *Server) handleRDUpdate(w http.ResponseWriter, r *http.Request) { // = delete-property), matching the RG path and the UG9 NOTE. rd.Props = applyPropsModify(rd.Props, patch.OverrideProps, patch.DeleteProps) + // The body declares delete_namespaces and the merge dropped it, so a + // modify carrying it answered 200 and changed nothing. Declaring two + // thirds of a props-modify envelope and silently discarding the rest + // tells the operator work happened when it did not. + deletePropNamespaces(rd.Props, patch.DeleteNamespaces) + if rgChange != "" { rd.ResourceGroupName = rgChange } diff --git a/pkg/rest/resource_groups.go b/pkg/rest/resource_groups.go index c718e7d5..56d4b173 100644 --- a/pkg/rest/resource_groups.go +++ b/pkg/rest/resource_groups.go @@ -632,8 +632,14 @@ func rgSelectFilterClearTable() map[string]func(*apiv1.AutoSelectFilter) { // so an override entry with an empty value DELETES the key — the // `linstor rg set-property g DrbdOptions/Resource/on-no-quorum` // (no value) = delete-property semantic the upstream UG9 NOTE pins. +// +// delete_namespaces is the third half of the same envelope, and the body +// declares it: merging only the first two answered a modify carrying +// delete_namespaces with 200 and changed nothing. The resource-definition +// modify had the same gap one file over. func mergeRGProps(existing, patch *apiv1.ResourceGroup) { existing.Props = applyPropsModify(existing.Props, patch.OverrideProps, patch.DeleteProps) + deletePropNamespaces(existing.Props, patch.DeleteNamespace) } // handleRGDelete drops a ResourceGroup. diff --git a/pkg/rest/volume_definitions.go b/pkg/rest/volume_definitions.go index 6227652f..4eb98d9e 100644 --- a/pkg/rest/volume_definitions.go +++ b/pkg/rest/volume_definitions.go @@ -999,13 +999,7 @@ func mergeVolumeDefinitionPatch(existing *apiv1.VolumeDefinition, patch *volumeD delete(existing.Props, k) } - for _, ns := range patch.DeleteNamespaces { - for k := range existing.Props { - if k == ns || (len(k) > len(ns) && k[:len(ns)] == ns && k[len(ns)] == '/') { - delete(existing.Props, k) - } - } - } + deletePropNamespaces(existing.Props, patch.DeleteNamespaces) } // handleVDDelete drops a VolumeDefinition under an RD. From a4934587a1e438ede38771e2cbc85d72f1113f46 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 7 Oct 2026 10:00:14 +0200 Subject: [PATCH 2/4] fix(clone,restore): accept linstor-csi's clone and restore calls, and resume their retries Clone rejected every request golinstor sends. The endpoint decoded with DisallowUnknownFields and declared five fields, so layer_list, which linstor-csi never omits, was a 400 before the handler ran, and every CSI clone-from-volume failed. The body now embeds golinstor's own props-modify triple, honours layer_list and resource_group on both clone paths, validates the stack the way rg modify does and checks the group exists, stores the stack in canonical case, and refuses external_name and volume_passphrases rather than accept and drop them. A requested stack that changes the layer set of a source with volumes is refused: every layer's bring-up writes to the device the clone just restored into. Restore refused linstor-csi on its first call. VolFromSnap creates the definition itself, restores its volumes, and only then asks for the resource restore, so the definition carries no restore marker and was taken for somebody else's. A definition without the marker is now taken when it is in exactly that state: its volumes the snapshot's, no replica, and the source's layer stack, since a layer the source did not have would write across the restored bytes. The marker is stamped only after every check that can still refuse it has passed, so a refusal leaves the caller's definition as it was. One with a live replica stays refused, since it may hold data. Both doors are idempotent the way CreateVolume has to be. A repeat over this operation's own leftover resumes it instead of answering AlreadyExists for good, and the leftover is judged in one place for both: replicas first, so one being torn down is refused before anything is written, then the parent group, then the mark a rollback leaves when it gives up. A finished clone or restore is judged by its volumes and, where the operation placed replicas, by holding one with a disk, so its replay answers success whatever happened to the source since. A diskless or tie-breaker replica holds no copy of the data: it never makes a leftover finished, and an operator's diskless replica already on a requested node is refused rather than counted as placed. The controller's tie-breaker witness there is promoted to the replica the restore asked for, as autoplace promotes one. A retry naming a different shape, or a leftover internal snapshot that fell behind the source, is refused rather than resumed. The marker is compared the way LINSTOR folds names. A retry that adopts a leftover marks it through the API server before writing anything, and the attempt that created it reads the mark before rolling back, so it never deletes what a retry already answered 201 for. A replay of a finished leftover takes the same mark before its 201, though it writes nothing else: a bare restore's replay can call finished what a placed restore of the same snapshot, still placing, is about to roll back. The handshake fails closed: a rollback that cannot write its own mark, or cannot read the adoption back, deletes nothing, and takes its in-progress mark back off, since a mark left on a definition nothing was taken from would refuse every retry while the answer said to retry. A mark that cannot be taken back is refused over with a correction that names the manual way out, and over a deleted group the correction starts with the group. A replica a stamp collided with is read from the API server, not the cache, which right after a delete began still serves it live. The prepared-target decision reads the definition, its volumes and its replicas from the API server too: linstor-csi's volume restore and the resource restore after it can land on different replicas of the server, and a cache behind the first refused a valid target or took one holding a live replica. The marker is stamped through the API server, so the reads in the same request that decide on it go there too; a cache that had not seen its own request's patch refused linstor-csi's first restore as somebody else's definition. Once the marker is on it stays: linstor-csi re-issues CreateVolume on a timeout, the retry finds the marker and places the replica, and the request that wrote the marker meets that replica as one already placed. Taking the marker back off there would leave a definition CSI was told is restored without one. The CLI door settles which nodes and pools the replicas go to before it writes the definition or the marker, so a request refused for its own shape leaves a prepared definition unmarked; a placement that fails after the marker is finished by running the same command again. It judges its own leftover with the assessment and gate order the REST door uses, so a finished restore is left alone rather than re-placed on a node the operator emptied, and a leftover whose rollback gave up is refused on both doors. A clone resume whose internal snapshot is gone is refused once the leftover holds a volume with a replica: retaking the snapshot would finish it from a later moment of the source than the one it was restored from. The leftover judgement counts replicas and reads volumes from the API server: a lagging informer still showing a replica being deleted as live called a restore finished over a definition that is going away. The CLI judges a definition already under the name before it plans anything from the source, so a finished restore is left alone once the source can no longer be planned from; it refuses a prepared definition whose group is gone before the marker goes on, and takes the adoption mark before it finishes a leftover, as the REST door does. A run that finds the marker already stamped on a target it judged prepared lost it to a concurrent run, and judges and finishes that run's definition as a leftover instead of placing over it with its own nodes. Refusals reach each client in the shape it decodes: python-linstor, which names itself in its User-Agent, keeps the CloneStarted object it decodes whatever the status, and every other caller, linstor-csi and any other golinstor client among them, gets the []ApiCallRc array. The CLI's rollback of a definition it created holds the same handshake as the REST one before it deletes anything: a definition another request adopted, and may already have answered for, is left to it, and one whose rollback mark could not be written or whose adoption could not be read is not deleted. Prop edits on a clone that would rewrite or remove a prop blockstor sets on the clone itself, the restore marker among them, are refused before anything is written: applied after the marker, they left the satellite to bring the volumes up blank. This also covers part of the known gap where a restored definition can lose its snapshot through override_props. A rollback still in progress is worded as one that may still finish on both doors, not as one that gave up. The marker is now written with a record of the volumes the snapshot held. The snapshot is what a retry judges a leftover against, and it does not always outlive the leftover; judged without it, a clone that had restored only some of its volumes read as complete. Once the snapshot is gone the record stands in for it. A resume rewrites the record from the snapshot it finishes from, which a clone may have retaken since the first attempt. rd create, rd modify and the CLI's rd property verbs refuse the props blockstor writes on a definition, as the clone door does: a delete of the Blockstor namespace mid-restore took the adoption mark with it. A CLI rollback that holds the handshake and then cannot delete takes its in-progress mark back off, so the next run finishes the definition instead of meeting a rollback nobody is running, and a CLI re-run over a finished restore says so. s vd restore stays strict on both doors, as on main: a volume already on the target is a refusal, so the losing run of two stops on the first volume with nothing of its own to unwind. Both doors also refuse a definition carrying a restore marker, which a restore or clone is filling and whose rollback would take what the volume restore answered for. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- config/manager/manager.yaml | 2 +- docs/cli-parity-known-deltas.md | 4 + internal/cli/props.go | 2 +- internal/cli/rollback_gone_test.go | 106 + internal/cli/snapshot.go | 701 +++++- internal/cli/snapshot_restore_doors_test.go | 250 +++ internal/cli/snapshot_restore_faults_test.go | 706 +++++++ internal/cli/snapshot_restore_marker_test.go | 66 + .../cli/snapshot_restore_prepared_test.go | 1571 ++++++++++++++ .../snapshot_restore_without_snapshot_test.go | 156 ++ internal/cli/write.go | 49 +- pkg/rest/adopted_leftover_rollback_test.go | 613 ++++++ pkg/rest/cache_retry.go | 5 + pkg/rest/controller_props.go | 1 + pkg/rest/delete_namespaces_doors_test.go | 126 ++ pkg/rest/group_server_owned_props_test.go | 219 ++ pkg/rest/kv_store.go | 11 +- pkg/rest/kv_store_test.go | 34 + pkg/rest/leftover_gates_test.go | 799 +++++++ pkg/rest/materialise_refusal_test.go | 194 ++ pkg/rest/node_connections.go | 13 +- pkg/rest/nodes.go | 5 +- pkg/rest/nodes_test.go | 7 + pkg/rest/props_delete_namespaces_test.go | 70 +- pkg/rest/props_modify.go | 15 +- pkg/rest/rd_clone.go | 1880 ++++++++++++++--- pkg/rest/rd_clone_client_envelope_test.go | 142 ++ pkg/rest/rd_clone_csi_retry_test.go | 485 +++++ pkg/rest/rd_clone_golinstor_shape_test.go | 204 ++ pkg/rest/rd_clone_idempotency_test.go | 299 +++ pkg/rest/rd_clone_layer_stack_test.go | 203 ++ pkg/rest/rd_clone_leftover_shape_test.go | 671 ++++++ pkg/rest/rd_clone_recorded_volumes_test.go | 182 ++ pkg/rest/rd_clone_replay_test.go | 898 ++++++++ .../rd_clone_resume_point_in_time_test.go | 376 ++++ pkg/rest/rd_clone_resume_test.go | 296 +++ pkg/rest/rd_clone_server_owned_props_test.go | 86 + pkg/rest/rd_clone_target_test.go | 332 +++ pkg/rest/rd_clone_validation_test.go | 254 +++ pkg/rest/rd_modify_server_owned_props_test.go | 149 ++ pkg/rest/resource_definitions.go | 170 +- pkg/rest/resource_group_extras.go | 21 +- pkg/rest/resource_groups.go | 93 +- pkg/rest/resource_modify.go | 3 + pkg/rest/restore_effective_layers_test.go | 181 ++ pkg/rest/rg_deleted_race.go | 499 ++++- pkg/rest/rg_deleted_race_round10_test.go | 31 +- pkg/rest/rg_deleted_race_round5_test.go | 11 +- pkg/rest/rg_deleted_race_round6_test.go | 82 +- pkg/rest/rg_deleted_race_round8_test.go | 24 +- pkg/rest/rg_deleted_race_round9_test.go | 49 +- .../rollback_handshake_failclosed_test.go | 664 ++++++ pkg/rest/rollback_identity_test.go | 280 +++ pkg/rest/server.go | 12 +- pkg/rest/snapshot_restore.go | 1126 +++++++++- pkg/rest/snapshot_restore_csi_flow_test.go | 240 +++ pkg/rest/snapshot_restore_idempotency_test.go | 438 ++++ .../snapshot_restore_prepared_cache_test.go | 224 ++ .../snapshot_restore_prepared_target_test.go | 174 ++ pkg/rest/snapshot_restore_replay_test.go | 183 ++ .../snapshot_restore_resume_gates_test.go | 133 ++ pkg/rest/snapshot_restore_test.go | 137 ++ ...snapshot_restore_volume_definition_test.go | 163 ++ .../snapshot_restore_without_snapshot_test.go | 334 +++ pkg/rest/sp_namespace_spelling_test.go | 114 + pkg/rest/spawn.go | 7 +- pkg/rest/storage_pool_definitions.go | 10 +- pkg/rest/storage_pools.go | 23 +- pkg/rest/store_faults_test.go | 742 +++++++ pkg/rest/vg_modify_golinstor_test.go | 129 ++ pkg/rest/volumes_per_resource.go | 9 +- pkg/store/adoption_set_test.go | 327 +++ .../k8s/prepared_target_uncached_test.go | 99 + pkg/store/k8s/resource_get_uncached_test.go | 99 + pkg/store/k8s/resources.go | 56 +- pkg/store/k8s/volume_definitions.go | 20 +- pkg/store/leftover.go | 302 +++ pkg/store/leftover_test.go | 150 ++ pkg/store/local_props.go | 232 +- pkg/store/namespace_covers_test.go | 65 + pkg/store/restore_faults_test.go | 483 +++++ pkg/store/restore_target.go | 740 +++++++ pkg/store/restore_target_test.go | 347 +++ pkg/store/rollback_hold_test.go | 294 +++ pkg/store/rollback_identity_test.go | 88 + stand/blockstor-apiserver-deploy.yaml | 2 +- stand/blockstor-deploy.yaml | 2 +- 87 files changed, 21114 insertions(+), 680 deletions(-) create mode 100644 internal/cli/rollback_gone_test.go create mode 100644 internal/cli/snapshot_restore_doors_test.go create mode 100644 internal/cli/snapshot_restore_faults_test.go create mode 100644 internal/cli/snapshot_restore_marker_test.go create mode 100644 internal/cli/snapshot_restore_prepared_test.go create mode 100644 internal/cli/snapshot_restore_without_snapshot_test.go create mode 100644 pkg/rest/adopted_leftover_rollback_test.go create mode 100644 pkg/rest/delete_namespaces_doors_test.go create mode 100644 pkg/rest/group_server_owned_props_test.go create mode 100644 pkg/rest/leftover_gates_test.go create mode 100644 pkg/rest/materialise_refusal_test.go create mode 100644 pkg/rest/rd_clone_client_envelope_test.go create mode 100644 pkg/rest/rd_clone_csi_retry_test.go create mode 100644 pkg/rest/rd_clone_golinstor_shape_test.go create mode 100644 pkg/rest/rd_clone_idempotency_test.go create mode 100644 pkg/rest/rd_clone_layer_stack_test.go create mode 100644 pkg/rest/rd_clone_leftover_shape_test.go create mode 100644 pkg/rest/rd_clone_recorded_volumes_test.go create mode 100644 pkg/rest/rd_clone_replay_test.go create mode 100644 pkg/rest/rd_clone_resume_point_in_time_test.go create mode 100644 pkg/rest/rd_clone_resume_test.go create mode 100644 pkg/rest/rd_clone_server_owned_props_test.go create mode 100644 pkg/rest/rd_clone_target_test.go create mode 100644 pkg/rest/rd_clone_validation_test.go create mode 100644 pkg/rest/rd_modify_server_owned_props_test.go create mode 100644 pkg/rest/restore_effective_layers_test.go create mode 100644 pkg/rest/rollback_handshake_failclosed_test.go create mode 100644 pkg/rest/rollback_identity_test.go create mode 100644 pkg/rest/snapshot_restore_csi_flow_test.go create mode 100644 pkg/rest/snapshot_restore_idempotency_test.go create mode 100644 pkg/rest/snapshot_restore_prepared_cache_test.go create mode 100644 pkg/rest/snapshot_restore_prepared_target_test.go create mode 100644 pkg/rest/snapshot_restore_replay_test.go create mode 100644 pkg/rest/snapshot_restore_resume_gates_test.go create mode 100644 pkg/rest/snapshot_restore_volume_definition_test.go create mode 100644 pkg/rest/snapshot_restore_without_snapshot_test.go create mode 100644 pkg/rest/sp_namespace_spelling_test.go create mode 100644 pkg/rest/store_faults_test.go create mode 100644 pkg/rest/vg_modify_golinstor_test.go create mode 100644 pkg/store/adoption_set_test.go create mode 100644 pkg/store/k8s/prepared_target_uncached_test.go create mode 100644 pkg/store/k8s/resource_get_uncached_test.go create mode 100644 pkg/store/leftover.go create mode 100644 pkg/store/leftover_test.go create mode 100644 pkg/store/namespace_covers_test.go create mode 100644 pkg/store/restore_faults_test.go create mode 100644 pkg/store/restore_target.go create mode 100644 pkg/store/restore_target_test.go create mode 100644 pkg/store/rollback_hold_test.go create mode 100644 pkg/store/rollback_identity_test.go diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 27adba42..99c3a0e1 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -107,4 +107,4 @@ spec: # The REST server's graceful-shutdown window is derived from the # detached rollback budget (pkg/rest/rg_deleted_race.go); this has to # outlive it, or a SIGTERM kills a compensation mid-cascade. - terminationGracePeriodSeconds: 20 + terminationGracePeriodSeconds: 21 diff --git a/docs/cli-parity-known-deltas.md b/docs/cli-parity-known-deltas.md index 698670d0..c6b55014 100644 --- a/docs/cli-parity-known-deltas.md +++ b/docs/cli-parity-known-deltas.md @@ -54,6 +54,10 @@ Row IDs match the command-catalogue indexes used by `cli-parity-refresh.sh` (see | 83 | `rg spawn-resources` on an over-committed RG (place_count > available nodes) | BEHAVIOR | permanent | Corner D1. Upstream LINSTOR fails `spawn-resources` SHORT when the RG's place_count exceeds the placeable node count: it returns `FAIL_NOT_ENOUGH_NODES` (ret_code 996, "Not enough available nodes") and places NOTHING. blockstor instead takes a DEFERRED, best-effort autoplace path: it spawns the RD + VDs, places as many diskful replicas as the topology allows (e.g. 3 of 7 on a 3-node cluster), and surfaces the shortfall as an INFO in the SUCCESS envelope (`resource definition spawned, autoplace deferred: : not enough candidate storage pools: placed N of M`, exit 0). The `RGRebalanceReconciler` (`internal/controller/rg_rebalance_controller.go`) then tops the replica count back up additively once more nodes appear. The over-commit `rg create` itself is ACCEPTED by both controllers (parity — never an early create-time error). Rationale: the deferred path lets the CSI external-provisioner retry loop converge on a created RD instead of hard-failing on a cluster that is one node away from satisfying the request, and is strictly additive (scale-down stays an explicit `r d`). Switching spawn back to the upstream 996 short-fail would be a deliberate API change, not a silent regression. Pinned by `pkg/rest/spawn_test.go::TestSpawnImpossiblePlacementReturnsActionableError` + `TestSpawnPartialFlagAllowsShortPlacement` (L1) and `tests/e2e/cli-matrix/rg-c-overcommit-spawn-defers.sh` (L6). | | 84 | `r c ` (tiebreaker→diskful promotion runs full SyncTarget, not skip-sync) | BEHAVIOR | permanent | Promoting a tiebreaker (diskless witness) to diskful runs a FULL SyncTarget on the promoted node instead of the upstream skip-sync. DELIBERATE: skip-sync would require BS to pre-stamp a matching day0 GI and let the fresh replica win the auto-primary election — but a diskful peer already holds data (`anyDiskfulPeerHasData == true`, `pkg/dispatcher/dispatcher.go`), so force-priming the fresh replica mints an UNRELATED Current UUID; the data-bearing peer then declines the handshake (`uuid_compare()=unrelated-data`, "Unrelated data, aborting!") and the pair wedges in mutual StandAlone that never auto-recovers (the respawn-StandAlone P0). BS therefore gates the auto-primary seed on `!anyDiskfulPeerHasData(peers)` and `!rdInitialized(rd)`: with a data-bearing peer present the promoted replica comes up Inconsistent and SyncTargets the real bytes off the peer. Data converges correctly; the only cost is a full resync of the volume — the safe trade vs. a StandAlone wedge. Upstream's skip-sync-on-promotion is not reproducible without reintroducing the wedge. Pinned by `tests/e2e/cli-matrix/r-c-over-tiebreaker-skip-sync.sh` (L6, asserts SyncTarget→UpToDate convergence + Bug 348 SyncSource shape) + the dispatcher auto-primary gate tests (`pkg/dispatcher/dispatcher_test.go`, respawn-StandAlone wedge regression). | | 85 | `r l` State column — bare `SyncSource`/`SyncTarget` literal (no `(NN%)` suffix) when OutOfSyncKib<=0 | WIRE_SHAPE | permanent | A replica actively resyncing renders `SyncTarget(NN%)` / `SyncSource(NN%)` WHILE there is data to copy. When the per-volume `OutOfSyncKib` is <= 0 (or the VD size is unknown), BS DELIBERATELY renders the BARE literal `SyncSource` / `SyncTarget` with NO `(NN%)` suffix: `withSyncPercent` (pkg/rest/resources.go, called from `annotateSyncProgress`) short-circuits on `outOfSyncKib<=0` rather than emit a misleading `(0%)`/`(100%)`. The drbd replication-state literal can be observed for a brief window before/after the peer-device OutOfSyncKib counter carries a non-zero value, so a bare Sync* token is a real and intended shape. The REGRESSION guarded against is the opposite — a TERMINAL `UpToDate` must never carry a `(NN%)` suffix (Bug A). So the contract is: a Sync* token may render bare OR with `(NN%)`, but a terminal disk_state never carries a percent. Pinned by `tests/e2e/cli-matrix/r-l-conns-shapes.sh` sub-test D (L6) + `annotateSyncProgress`/`withSyncPercent` unit coverage (pkg/rest, Bug 348 + Bug A/B). | +| 86 | `rd clone --external-name` / `--volume-passphrase` / a `--layer-list` that changes the source's layer set | MISSING_FEATURE | permanent | `external_name` gives the clone an identity of its own upstream and `volume_passphrases` carries the LUKS keys for its volumes; blockstor names a clone by `name` and takes its keys from the cluster passphrase, so honouring neither is possible and both are refused with an explicit 501 rather than accepted and dropped. Every refusal on this endpoint, the 501 included, goes out in the shape the caller decodes: the CloneStarted object for python-linstor, which names itself `PythonLinstor/…` and reads the clone answer as that object whatever the status, and an ApiCallRc array for every other client, since golinstor decodes a non-2xx body as nothing else (linstor-csi's 404s reach it bare either way) — a clone that reports success under a different name, or with keys the caller does not hold, is the failure mode `src_snap_name` is already refused for. `layer_list` IS honoured, with one refusal: on a source that has volumes the requested stack must be the source's, compared as a set (order is the stack's own and case folds). The clone data plane restores the source's bytes and brings the layer stack up over them, in that order, and every layer's bring-up writes to the device — LUKS formats a device carrying no header, DRBD stamps `meta-disk internal` metadata with `create-md --force` — so a layer added here lands across the bytes just restored and the clone reports COMPLETE over them, while a layer dropped leaves the target reading data the missing layer wrote. A source with no recorded stack means the upstream default `[DRBD, STORAGE]`, which is what linstor-csi sends, so the CSI clone path is unaffected. Upstream clones the layer list with the volume, re-encrypting LUKS under the cluster key. A volume-less source has no bytes to lose and keeps taking any stack. A clone resumed over a leftover also keeps that leftover's shape: a retry naming a different `resource_group` or stack is refused rather than answered 201 with the request's shape dropped, and a leftover internal snapshot that has fallen behind the source (a volume added, or resized) is refused rather than cloned from. Pinned by `pkg/rest/rd_clone_golinstor_shape_test.go` + `pkg/rest/rd_clone_client_envelope_test.go` + `pkg/rest/rd_clone_layer_stack_test.go` + `pkg/rest/rd_clone_resume_test.go` + `pkg/rest/rd_clone_leftover_shape_test.go` + `pkg/rest/rd_clone_server_owned_props_test.go` + `pkg/rest/rd_modify_server_owned_props_test.go` + `internal/cli/snapshot_restore_prepared_test.go` (L1) and `tests/e2e/cli-matrix/rd-clone-retry-semantics.sh` (L6, stand-only: `make e2e-list` covers `tests/e2e/*.sh` and no CI lane runs the cli-matrix cells) + `tests/operator-harness/replay/rd-clone-retry-semantics.yaml` (L7). `rd modify` and `rd set-property` / `delete-property` refuse the same server-owned props, with 400 and an error respectively, since a `delete_namespaces` of `Blockstor` mid-restore took the adoption mark with it, and `rd create` refuses a definition carrying one with 400. Prop edits on a clone (`override_props`, `delete_props`, `delete_namespaces`) that would rewrite or remove a prop blockstor sets on the clone itself, the restore marker, the record of the snapshot's volumes, the rollback mark or the adoption mark, are refused with 400 before anything is written, rather than applied after the marker and leaving the satellite to bring the volumes up blank; linstor-csi's own `delete_props` edit touches none of them. | +| 87 | `rd clone` / `s resource restore` retry over an unfinished target | BEHAVIOR | permanent | This is the data path: a clone of a source with volumes, and every restore. A clone of a volume-less source is a plain copy with no marker, so a repeat of it stays the already-exists refusal upstream gives. Both endpoints stamp `BlockstorRestoreFromSnapshot` with the target definition and hydrate its volumes and place its replicas afterwards, so a target carrying the marker says the operation STARTED, not that it finished. A repeat under the same name RESUMES it — every step tolerates an object a previous attempt already created — and says so (`… completed on retry`); upstream has no equivalent notion, a repeat there is a plain already-exists failure. This is what makes CSI's CreateVolume idempotent: external-provisioner has no other way to make progress after a partial failure, and answering success on the marker alone would leave CSI seeing a ready volume nothing ever finishes. A definition under that name that carries no marker is taken when it is in the state LINSTOR's own sequence leaves (`rd c`, `s vd restore`, then `s rsc restore`): its volumes exactly the snapshot's and no live replica. linstor-csi restores every volume that way, creating the definition itself first, and the marker is stamped on it at that point. What stays a refusal, each before anything is written to the definition: any other definition under that name this operation did not create and that is not prepared, meaning its volumes are not the snapshot's or it holds a live replica (409); a prepared definition whose layer stack is not the snapshot source's, compared as a set, since the restore would put the source's bytes under layers it never had or without layers it had (409); a prepared definition, or a leftover still to be finished or one the request places replicas into, whose every replica is still being deleted, which placing over would race (409; a bare REST restore's replay over a finished leftover still answers 201 while its snapshot exists, since it places none); a prepared definition or a leftover parented to a resource group that no longer exists (409, or 500 asking for a retry when the group cannot be read); this operation's own leftover carrying the DELETE flag, since finishing it races the tear-down reaping what it writes (409); a leftover whose earlier attempt's rollback gave up, which may hold less than the operation intended, with the way out named (409); a leftover holding a volume the snapshot never recorded beside one it did (409); and a leftover whose creating attempt has started rolling it back (409). Placing a replica, after the definition and its volumes are written, refuses an operator's diskless replica already on a node the request places onto, which holds no copy of the data and is not counted as placed: a 409 on a retry over this operation's own leftover, and on a fresh restore a failed placement, which rolls the new definition back and answers 500, neither with a FAIL_EXISTS band. The controller's tie-breaker witness there is promoted to the replica the request asked for instead, as autoplace promotes one, and its volumes are restored from the snapshot like any other replica of the target; a witness no storage pool can be found for is left as it is and refused the same way, naming `linstor resource create --storage-pool ` to give it one. Once the marker is on a prepared definition it stays: a concurrent retry that finds it resumes and places the replica, and the request that wrote it meets that replica as one already placed, never as a reason to take the marker back off. The window this leaves is a replica placed by hand between the judgement and the marker, which is then judged as part of the restore; on a target prepared for exactly this restore nothing else places one in that moment. A replica the operation would place that is still being deleted on its node fails the request rather than counting as placed (a 409 over a leftover this request adopted, and a 500 with the rollback's answer over one it created), with no FAIL_EXISTS_RSC band on either. Two restores of different snapshots into one prepared definition do not both take it: the second finds the first's marker and is refused. The CLI door settles the parent group, and the nodes and pools the replicas go to, before it writes the definition or the marker, so a request refused for its own shape, or over a prepared definition whose group is gone, leaves that definition unmarked; and a CLI placement that fails after the marker is finished by running the same command again, unless a replica already landed: the re-run then judges it finished, so the failure names the replicas that did not land and the `blockstor resource create` that places them. The CLI judges its own leftover with the same assessment and gate order the REST door uses (a tear-down, then a volume the restore would not have written, then a parent group that is gone, then a rollback that gave up): a leftover with the snapshot's volumes and one replica with a disk is finished and left alone rather than re-placed on a node the operator emptied, and only one with no such replica gets its placement finished, under the same adoption mark the REST door takes, so a rollback by the attempt that created it yields. A definition already under the name is judged before anything about the source is planned, so a finished restore is left alone even once the source can no longer be planned from. Replicas are counted past the cache on both doors, and a rollback still in progress is refused as one still running rather than as one that gave up. A clone resume whose internal snapshot is gone is refused once the leftover holds a volume with a replica that has a disk, since finishing it from a new snapshot would join two moments of the source; a leftover with no volume or no replica is finished from the new snapshot like a fresh clone, unless its volumes do not fit the source as it is now (one smaller than the source's, or one the source no longer has), which is refused with `delete and clone again` before any snapshot is taken. The volumes and replicas of a prepared definition, and a colliding replica, are read from the API server, not a cache, and so is the definition the request judges right after stamping its marker: linstor-csi's volume restore and the resource restore after it can be served by different replicas of the server. A finished clone or restore is judged by its volumes and, where the operation placed replicas, by holding one with a disk, never by where they are now: a replay after a node was evacuated or the target scaled down answers success and leaves placement alone. A finished clone is a copy of a point-in-time and its replay answers the same way whatever happens to the source afterwards — including a `vd set-size`, which a guard comparing the internal snapshot to the live source turned into a permanent refusal of every retry. Pinned by `pkg/rest/rd_clone_idempotency_test.go` + `pkg/rest/rd_clone_replay_test.go` + `pkg/rest/rd_clone_leftover_shape_test.go` + `pkg/rest/snapshot_restore_idempotency_test.go` + `pkg/rest/snapshot_restore_replay_test.go` + `pkg/rest/snapshot_restore_csi_flow_test.go` + `pkg/rest/snapshot_restore_prepared_target_test.go` + `pkg/rest/snapshot_restore_prepared_cache_test.go` + `pkg/rest/snapshot_restore_resume_gates_test.go` + `pkg/rest/leftover_gates_test.go` + `pkg/rest/rollback_handshake_failclosed_test.go` + `pkg/store/restore_target_test.go` + `internal/cli/snapshot_restore_prepared_test.go` (L1) and `tests/e2e/cli-matrix/rd-clone-retry-semantics.sh` (L6, stand-only: `make e2e-list` covers `tests/e2e/*.sh` and no CI lane runs the cli-matrix cells) + `tests/operator-harness/replay/rd-clone-retry-semantics.yaml` (L7). Every marker is written with `Blockstor/RestoreVolumes`, the volumes the snapshot held, so a leftover whose snapshot is gone (an operator deleted it) is still judged against them: a clone that restored only some of its volumes is unfinished, not complete. A leftover from before the record existed carries none and is judged on its own volumes, as before. A CLI rollback that could not delete takes its in-progress mark back off so the next run finishes the definition, and a CLI re-run over a finished restore says so on stdout. The CLI's rollback of a definition it created holds the same handshake as REST before deleting: a definition another request adopted is left to it, and one whose mark could not be written or whose adoption could not be read is not deleted. A rollback after a failed clone or restore also leaves a definition that already reads as finished, holding every volume and, where the operation places replicas, one replica with a disk: the clone status poll answers such a definition complete without writing anything, which can happen while the attempt that created it is still stamping its last replica, so deleting it afterwards took a volume the driver had been told exists. A replay takes the adoption mark before its 201, so the rollback yields to it like to any adopter, whatever the request placed. The rollback over a group deleted under the operation is the exception: it removes a finished definition too, since one parented to a group that is gone is what it exists to remove, so a status poll that answered complete a moment before it starts is followed by the definition going away; a POST replay's mark makes it yield. That attempt fails, and its retry answers for what it left; a restore that asked for replicas on named nodes names the ones it never placed, with the `linstor resource create` that places them, since the retry judges the definition finished and places nothing. A clone's prop edits (`override_props`, `delete_props`, `delete_namespaces`) land with the definition when it is created, so no leftover carries a prop the caller asked to remove: linstor-csi removes `Aux/csi-provisioning-completed-by` on every clone and takes a definition carrying it, with volumes of the right size, as one already provisioned. The clone status poll asks a finished target what the POST's replay asks before its 201: one carrying the DELETE flag, parented to a group that is gone, or carrying a rollback in progress or one that gave up is answered 404, which sends the driver back to the POST that refuses it, and a group that cannot be read is a 500. A replay of a finished clone is answered before the source is read, so it survives the source being deleted or losing its volumes. A retry after the snapshot was deleted, the cleanup that follows a finished restore, is judged against the volumes the definition recorded when it was restored: a finished one answers 201 (the CLI exits 0) without reading the snapshot or the source, an unfinished one is refused with `delete and restore it from another snapshot`, and a target restored before the record existed keeps the 404 it got. A restored or cloned definition is stamped with the source's data-plane layer stack (its own stored stack, or DRBD over STORAGE when it has none): the dispatcher reads only the definition's stack, so a target group with another stack never changed how the bytes are brought up, but the control plane reads the group's stack for a definition stored without one, and the stamp keeps the two in step. A REST replay of a finished restore checks its node list only while the snapshot exists; once it is gone the replay is judged by the target alone. A bare restore counts as finished without a replica only while its snapshot exists: once the snapshot is gone, a target with no replica holds its data nowhere and is refused with `delete and restore it from another snapshot`, as the CLI refuses it. A clone that names no layer stack takes the source's and needs the cluster passphrase when that stack carries LUKS, as one that names it does; the check runs after the replay of a finished clone, which writes no new LUKS volume. A target already under the name is judged, hydrated and placed under the name it is stored with, whatever case the request spells it in: its replicas are selected by that name exactly, and judged under another spelling a finished clone or restore read as having none; a read of that name that fails is the 500 below. A target that cannot be read is answered 500 asking for a retry, not taken as absent: the create that followed collided and was adopted past every judgement of a leftover. The CLI refuses a snapshot that recorded no volumes before it writes or resumes anything, as the REST door does. A rollback that leaves a definition whole and cannot take its in-progress mark back off says so and names `linstor resource-definition set-property BlockstorRollbackAbandoned` with no value (python-linstor's delete; the blockstor CLI also has `delete-property`), which `rd modify` (`delete_props`) and the CLI accept as the one edit of a server-owned prop an operator may make, and only while the mark sits on a definition the rollback left whole (`in-progress`, or `snapshots` / `read-snapshots`, where it stopped before touching anything): one naming a later step sits on a definition the rollback took partly apart, and stays; setting the mark stays refused, and an empty `override_props` value of it is read as the delete. The refusal over an in-progress mark names the clear only once the rollback has had its time; a clear while it still runs stops that rollback before it deletes anything, since the rollback checks its own mark is still there before its first delete. A CLI rollback that leaves the target in place names the `blockstor snapshot resource restore` that finishes it, including for `rd clone`, whose own re-run is refused over a target that exists. | +| 88 | `rg vg m --gross` / `rg vg m` with `flags` | MISSING_FEATURE | permanent | Upstream's one volume-group flag is `GROSS_SIZE`, which sizes a spawned volume gross. Nothing in blockstor sizes a spawned volume gross, so the volume-group modify door answers 501 to any `flags` in the body and writes nothing, the props half included, rather than a 200 for a request that changes nothing. | +| 89 | `delete_namespaces` on every props-modify door | BEHAVIOR | permanent | Not a divergence; recorded so it is not reopened. As upstream's `PropsContainer.removeNamespace` does, a namespace removes the keys below it and leaves a key spelled exactly as the namespace: `DrbdOptions` takes `DrbdOptions/Net/protocol` and keeps `DrbdOptions`. Trailing slashes name the same namespace. Every door and every guard in front of one read it through the same helper. | | 07 | `rd l --resource-definitions ` (Layers column on a placed RD) | WIRE_SHAPE | permanent | Same `stampRDLayerDataFromStack` behaviour as row 81 — the flag-qualified `rd l --resource-definitions ` catalogue cell (harness index 07) the bare `rd l` whitelist string does not literally cover. BS re-synthesises `layer_data: [{"type":"DRBD"},{"type":"STORAGE"}]` from `Spec.LayerStack` on every RD read, so the python CLI's Layers column renders `DRBD,STORAGE` even for a fresh `parity-rd` with only a volume-definition; upstream 1.33.2 leaves the column blank until DRBD layer data is actually allocated. BLOCKSTOR_SUPERSET, operator-friendly, and linstor-csi / piraeus-operator do not gate on RD `layer_data`. See row 81 for the full rationale and the L1/L3 pins (`stampRDLayerDataFromStack`, `tests/contract/normalize_test.go::TestNormalizeRDLayerDataDropped`). | | 33 | `s d ` (idempotent snapshot delete) | WIRE_SHAPE | permanent | BLOCKSTOR idempotent-delete envelope. Deleting a snapshot definition that does not exist returns `SUCCESS: snapshot already absent: ` (the desired-state delete is satisfied); upstream LINSTOR returns `WARNING: Snapshot definition of resource not found`. Both exit 0. Deliberate: a CSI `DeleteSnapshot` / operator retry MUST be idempotent, so "already gone" is a success, not a warning. Mirrors the resource-delete idempotency in row 42. Pinned by the snapshot-delete REST handler's already-absent path (`pkg/rest/snapshots.go`). | diff --git a/internal/cli/props.go b/internal/cli/props.go index b1e85a63..d4ed50ad 100644 --- a/internal/cli/props.go +++ b/internal/cli/props.go @@ -88,7 +88,7 @@ var storagePoolProps = withNamedKeyGuard(objectProps("storage pool", 2, // (node // The REST path has refused these since it was written; this CLI edits // the same bag directly, so the rule has to hold here too. validate.StoragePoolPropEdit, -), validate.StoragePoolPropNamed) +), func(key string, _ bool) error { return validate.StoragePoolPropNamed(key) }) // resourceGroupProps accesses a group's property bag. // diff --git a/internal/cli/rollback_gone_test.go b/internal/cli/rollback_gone_test.go new file mode 100644 index 00000000..d2b673ff --- /dev/null +++ b/internal/cli/rollback_gone_test.go @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cli + +import ( + "context" + "errors" + "sync/atomic" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +var errPlacementFailed = errors.New("placement failed") + +// markFindsNothing answers the gone-th definition patch NotFound, while the +// backend holds a definition under the name: the one the rollback meant went, +// and another writer put a new one there. +type markFindsNothing struct { + store.ResourceDefinitionStore + + patches *atomic.Int32 + gone int32 +} + +func (r markFindsNothing) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + if r.patches.Add(1) == r.gone { + return store.ErrNotFound + } + + return r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) //nolint:wrapcheck // test double +} + +type markFindsNothingStore struct { + store.Store + + patches *atomic.Int32 + gone int32 +} + +func (s markFindsNothingStore) ResourceDefinitions() store.ResourceDefinitionStore { + return markFindsNothing{ResourceDefinitionStore: s.Store.ResourceDefinitions(), patches: s.patches, gone: s.gone} +} + +// A restore rollback whose definition is gone deletes nothing by name, and +// answers with the restore's own failure alone, since there was nothing left +// to roll back: whether it finds the definition gone on the mark that starts +// the rollback or on the one that takes it apart. +func TestRestoreRollbackWhoseDefinitionGoesBeforeItsDeleteDeletesNothing(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + gone int32 + }{ + {name: "gone when the rollback starts", gone: 1}, + {name: "gone before the first delete", gone: 2}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "pvc-regone"}); err != nil { + t.Fatalf("seed: %v", err) + } + + // The flags the rollback's advice reads, so a regression fails on + // this case's own assertions rather than a nil dereference that + // takes the sibling case down with it. + run := &runContext{ + Store: markFindsNothingStore{Store: backend, patches: new(atomic.Int32), gone: tc.gone}, + Flags: &flagSet{Values: map[string]string{}}, + } + + err := rollbackRestore(ctx, run, "pvc-regone", nil, errPlacementFailed) + if err == nil || err.Error() != errPlacementFailed.Error() { + t.Errorf("rollback = %v, want the restore's own failure and nothing else", err) + } + + if _, err := backend.ResourceDefinitions().Get(ctx, "pvc-regone"); err != nil { + t.Errorf("the definition now under the name was deleted: %v", err) + } + }) + } +} diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index a1b940d8..59135b9f 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -34,13 +34,6 @@ import ( "github.com/cozystack/blockstor/internal/cli/command" ) -// restoreFromSnapshotProp routes the satellite to -// RestoreVolumeFromSnapshot instead of CreateVolume. Its value is -// `:`; the satellite splits on the colon. -// It is stamped on the definition rather than per replica because -// every replica of a restored definition clones from the same source. -const restoreFromSnapshotProp = "BlockstorRestoreFromSnapshot" - // errRollbackUnsupported explains a deliberate omission. // // In-place rollback destroys every snapshot newer than the target @@ -309,23 +302,78 @@ func snapshotRestoreResource(ctx context.Context, run *runContext) error { return err } - src, err := run.Store.ResourceDefinitions().Get(ctx, args.fromResource) + // A definition already under the name is judged before anything about + // the source is read or planned: a finished restore is left alone even + // once the source or its snapshot is gone. + exists, handled, err := replayOwnRestore(ctx, run, args) + if handled || err != nil { + return err + } + + src, snap, err := readRestoreSource(ctx, run, args) if err != nil { - return fmt.Errorf("get resource definition %s: %w", args.fromResource, err) + return err } - snap, err := run.Store.Snapshots().Get(ctx, args.fromResource, args.fromSnapshot) + // A snapshot that recorded no volumes has nothing to restore, and is + // refused before anything is written or resumed, as the REST door refuses + // it: a resume over a leftover of one hydrated nothing and placed + // replicas, exiting 0 over a definition with no volume. + if len(snap.VolumeDefinitions) == 0 { + return fmt.Errorf("%s captured no volumes to restore onto %s: %w", + snap.Name, args.toResource, errNothingToRestore) + } + + if exists { + return restoreIntoExistingTarget(ctx, run, args, &snap, nil) + } + + // Everything about where the replicas go is settled before anything is + // written: a node that does not hold the snapshot, or a pool that cannot + // be inferred, refused after the definition or its marker had landed. + planned, err := planRestoredReplicas(ctx, run, args.fromResource, args.toResource, &snap) if err != nil { - return fmt.Errorf("get snapshot %s of %s: %w", args.fromSnapshot, args.fromResource, err) + return err + } + + def := newRestoredDefinition(args.toResource, &src, &snap) + + err = run.Store.ResourceDefinitions().Create(ctx, def) + if errors.Is(err, store.ErrAlreadyExists) { + return restoreIntoExistingTarget(ctx, run, args, &snap, err) + } + + if err != nil { + return fmt.Errorf("create resource definition %s: %w", def.Name, err) + } + + // The definition is this command's own, but a concurrent run may already + // have adopted it and be hydrating the same volumes. A volume that is + // already there at the snapshot's size is the restore's, not a collision + // to unwind: taking it back would delete what the adopter answered for. + err = hydrateMissingVolumes(ctx, run, def.Name, &snap) + if err != nil { + return rollbackRestore(ctx, run, def.Name, planned, err) + } + + err = stampRestoredReplicas(ctx, run, planned) + if err != nil { + return rollbackRestore(ctx, run, def.Name, planned, err) } + return nil +} + +// newRestoredDefinition is the definition a restore creates: the source's +// group and stack, the snapshot's props, and this restore's marker. +func newRestoredDefinition(rdName string, src *apiv1.ResourceDefinition, snap *apiv1.Snapshot) *apiv1.ResourceDefinition { def := &apiv1.ResourceDefinition{ - Name: args.toResource, + Name: rdName, // The parent group carries over: it drives later placement and // property inheritance, and a restored definition with a blank // group breaks the restore-then-list workflow. ResourceGroupName: src.ResourceGroupName, - LayerStack: src.LayerStack, + LayerStack: store.EffectiveLayerStack(src.LayerStack), Props: store.TravellingProps(snap.Props), } @@ -333,30 +381,330 @@ func snapshotRestoreResource(ctx context.Context, run *runContext) error { def.Props = store.TravellingProps(src.Props) } - if def.Props == nil { - def.Props = map[string]string{} + // Both halves off the stored objects, never off what the operator typed: + // LINSTOR folds name case, and a REST retry over this leftover compares + // the marker it finds against one built from the stored snapshot, while + // the placer looks the source half up as a store key. + def.Props = store.WithRestoreMarker(def.Props, snap) + + return def +} + +// restoreIntoExistingTarget finishes a restore over a definition that is +// already there, in one of two states, and refuses anything else with the +// create's own error. +// +// A target the operator prepared the way LINSTOR documents it (`rd c`, then +// `s vd restore`, then this command) is marked and placed; see +// store.PreparedRestoreTarget. The REST door takes the same state, which is +// also how linstor-csi restores. +// +// This restore's own leftover is judged the way the REST door judges it; see +// store.JudgeRestoreLeftover. The marker goes on before placement, so a +// placement that failed after it is finished by running the same command +// again rather than by deleting the definition, while a finished one is left +// alone. +// +// The definition is the operator's, so a failure past this point leaves it in +// place rather than deleting it. +func restoreIntoExistingTarget( + ctx context.Context, run *runContext, args restoreArgs, snap *apiv1.Snapshot, createErr error, +) error { + rdName := args.toResource + + existing, err := run.Store.ResourceDefinitions().Get(ctx, rdName) + if err != nil { + return fmt.Errorf("get resource definition %s: %w", rdName, err) } - def.Props[restoreFromSnapshotProp] = args.fromResource + ":" + snap.Name + if createErr == nil { + createErr = fmt.Errorf("%w: resource definition %q", store.ErrAlreadyExists, existing.Name) + } - err = run.Store.ResourceDefinitions().Create(ctx, def) + progress, own, err := store.JudgeRestoreLeftover(ctx, run.Store, existing.Name, snap) if err != nil { - return fmt.Errorf("create resource definition %s: %w", def.Name, err) + return fmt.Errorf("restore into %s: %w", existing.Name, err) + } + + if own { + return finishRestoreLeftover(ctx, run, args, existing.Name, snap, progress) + } + + // A layer stack that is not the source's, or replicas still being + // deleted, refuse here, before the marker goes on a definition that is + // the operator's; see store.PreparedRestoreTarget. + prepared, err := store.PreparedRestoreTarget(ctx, run.Store, &existing, snap) + if err != nil { + return fmt.Errorf("restore into %s: %w", existing.Name, err) + } + + if !prepared { + return fmt.Errorf("create resource definition %s: %w", rdName, createErr) + } + + // The group and the placement are settled before the marker, the way the + // REST door settles them: a marker left on a definition parented to a + // group that is gone, or one no node can be placed on, is a definition the + // next run refuses over. + err = store.RestoreTargetGroupSurvived(ctx, run.Store, &existing) + if err != nil { + return fmt.Errorf("restore into %s: %w", existing.Name, err) + } + + planned, err := planRestoredReplicas(ctx, run, args.fromResource, existing.Name, snap) + if err != nil { + return err + } + + adopted, err := store.AdoptPreparedRestoreTarget(ctx, run.Store, existing.Name, snap) + if err != nil { + return fmt.Errorf("restore into %s: %w", existing.Name, err) + } + + // Not adopted with no error means the same marker landed between the + // judgement above and the stamp: a concurrent run of this restore got + // there first. Its definition is judged and finished as a leftover, the + // way the REST door does, rather than placed over with this run's nodes. + if !adopted { + return finishConcurrentRestore(ctx, run, args, existing.Name, snap, createErr) + } + + return placeOverExisting(ctx, run, planned) +} + +// finishConcurrentRestore judges a target another run of the same restore +// marked first, and finishes it as a leftover of this restore. +func finishConcurrentRestore( + ctx context.Context, run *runContext, args restoreArgs, rdName string, snap *apiv1.Snapshot, createErr error, +) error { + progress, own, err := store.JudgeRestoreLeftover(ctx, run.Store, rdName, snap) + if err != nil { + return fmt.Errorf("restore into %s: %w", rdName, err) + } + + if !own { + return fmt.Errorf("create resource definition %s: %w", rdName, createErr) + } + + return finishRestoreLeftover(ctx, run, args, rdName, snap, progress) +} + +// readRestoreSource reads the definition and the snapshot a restore is taken +// from. +func readRestoreSource(ctx context.Context, run *runContext, args restoreArgs) (apiv1.ResourceDefinition, apiv1.Snapshot, error) { + src, err := run.Store.ResourceDefinitions().Get(ctx, args.fromResource) + if err != nil { + return src, apiv1.Snapshot{}, fmt.Errorf("get resource definition %s: %w", args.fromResource, err) } - err = hydrateVolumes(ctx, run, def.Name, &snap) + snap, err := run.Store.Snapshots().Get(ctx, args.fromResource, args.fromSnapshot) if err != nil { - return rollbackRestore(ctx, run, def.Name, err) + return src, snap, fmt.Errorf("get snapshot %s of %s: %w", args.fromSnapshot, args.fromResource, err) } - err = placeRestored(ctx, run, args.fromResource, def.Name, &snap) + return src, snap, nil +} + +// sayAlreadyRestored reports a re-run over a finished restore. Said out loud: +// a re-run that exits 0 with nothing on the screen reads as a restore that +// just happened. +func sayAlreadyRestored(run *runContext, rdName, srcRD, snapName string) error { + _, err := fmt.Fprintf(run.Out, "%s is already restored from %s/%s, nothing to do\n", rdName, srcRD, snapName) if err != nil { - return rollbackRestore(ctx, run, def.Name, err) + return fmt.Errorf("write output: %w", err) } return nil } +// errSnapshotGoneUnfinished refuses a restore whose own leftover never +// finished and whose snapshot is gone: what is missing can only come from the +// snapshot. +var errSnapshotGoneUnfinished = errors.New("the snapshot is gone and the restore into the target never finished") + +// replayOwnRestore answers a re-run over this restore's own leftover before +// anything about the source is read. Deleting the snapshot, or the source, +// once the restore finished is ordinary cleanup, and a re-run after it is +// still a re-run: a finished leftover is judged against the volumes it +// recorded and left alone, the way the REST replay answers it. An unfinished +// one with its snapshot gone is refused with the way out. It reports whether +// a definition is under the name, and whether it answered; not answered hands +// the request to the path that reads the source. +func replayOwnRestore(ctx context.Context, run *runContext, args restoreArgs) (bool, bool, error) { + existing, err := run.Store.ResourceDefinitions().Get(ctx, args.toResource) + if errors.Is(err, store.ErrNotFound) { + return false, false, nil + } + + if err != nil { + return false, true, fmt.Errorf("get resource definition %s: %w", args.toResource, err) + } + + if !strings.EqualFold(existing.Props[store.RestoreFromSnapshotProp], args.fromResource+":"+args.fromSnapshot) { + return true, false, nil + } + + handled, err := judgeOwnRestore(ctx, run, args, &existing) + + return true, handled, err +} + +// judgeOwnRestore is replayOwnRestore over a definition that carries this +// restore's marker. +func judgeOwnRestore( + ctx context.Context, run *runContext, args restoreArgs, existing *apiv1.ResourceDefinition, +) (bool, error) { + var snapshot *apiv1.Snapshot + + snap, err := run.Store.Snapshots().Get(ctx, args.fromResource, args.fromSnapshot) + + switch { + case err == nil: + snapshot = &snap + case errors.Is(err, store.ErrNotFound): + if _, recorded := store.RecordedRestoreVolumes(existing.Props); !recorded { + return true, fmt.Errorf("get snapshot %s of %s: %w", args.fromSnapshot, args.fromResource, err) + } + default: + return true, fmt.Errorf("get snapshot %s of %s: %w", args.fromSnapshot, args.fromResource, err) + } + + progress, own, err := store.JudgeRestoreLeftoverOf(ctx, run.Store, existing.Name, + args.fromResource, args.fromSnapshot, snapshot) + if err != nil { + return true, fmt.Errorf("restore into %s: %w", existing.Name, err) + } + + if !own { + return false, nil + } + + if progress == store.LeftoverFinished { + return true, answerFinishedLeftover(ctx, run, existing.Name, args.fromResource, args.fromSnapshot) + } + + if snapshot == nil { + return true, fmt.Errorf("%w: %s/%s, %s; delete %s and restore it from another snapshot", + errSnapshotGoneUnfinished, args.fromResource, args.fromSnapshot, existing.Name, existing.Name) + } + + return false, nil +} + +// finishRestoreLeftover acts on this restore's own leftover as judged by +// store.JudgeRestoreLeftover. A finished one is left as it is, the way the REST +// replay answers it, and nothing about the source is planned for it: +// re-placing it would put a replica back on a node the operator emptied, +// restored from the point-in-time, and a source that can no longer be planned +// from must not fail a restore that is already done. An unfinished one is +// claimed the way the REST door claims a leftover, so a rollback of the +// attempt that created it yields rather than deleting what this run finishes, +// then gets the volumes it is still missing and its placement. +func finishRestoreLeftover( + ctx context.Context, run *runContext, args restoreArgs, rdName string, snap *apiv1.Snapshot, + progress store.LeftoverProgress, +) error { + // Done before, and left alone. Said out loud: a re-run that exits 0 + // with nothing on the screen reads as a restore that just happened. + if progress == store.LeftoverFinished { + return answerFinishedLeftover(ctx, run, rdName, snap.ResourceName, snap.Name) + } + + planned, err := planRestoredReplicas(ctx, run, args.fromResource, rdName, snap) + if err != nil { + return err + } + + err = store.ClaimAdoptedLeftover(ctx, run.Store, rdName, snap) + if err != nil { + return fmt.Errorf("restore into %s: %w", rdName, err) + } + + err = hydrateMissingVolumes(ctx, run, rdName, snap) + if errors.Is(err, errVolumeGoneOnReadBack) { + // Nothing rolls a resume back, so the way out is said here. + return fmt.Errorf("%w; %s", err, restoreAgain(run)) + } + + if err != nil { + return err + } + + return placeOverExisting(ctx, run, planned) +} + +// answerFinishedLeftover reports a finished leftover of this restore as +// restored, after taking the adoption mark the way the REST replay does +// (store.ClaimAdoptedLeftover with no snapshot, recording nothing). The mark is +// what tells the attempt that created the definition, if it is still rolling +// back, that somebody was told it is done; without it that rollback deletes +// the definition this run just vouched for. A creator already rolling back +// refuses the claim, and the run then reports the restore as not done. +func answerFinishedLeftover(ctx context.Context, run *runContext, rdName, fromResource, fromSnapshot string) error { + err := store.ClaimAdoptedLeftover(ctx, run.Store, rdName, nil) + if err != nil { + return fmt.Errorf("restore into %s: %w", rdName, err) + } + + return sayAlreadyRestored(run, rdName, fromResource, fromSnapshot) +} + +// hydrateMissingVolumes creates the snapshot's volumes a definition of this +// restore's does not hold yet, and keeps the ones it does: a volume already +// there at the snapshot's size is the restore's own, whether an earlier run or +// a concurrent one wrote it, and so is one grown since; a smaller one is not, +// and is refused. +func hydrateMissingVolumes(ctx context.Context, run *runContext, rdName string, snap *apiv1.Snapshot) error { + for i := range snap.VolumeDefinitions { + svd := &snap.VolumeDefinitions[i] + + err := run.Store.VolumeDefinitions().Create(ctx, rdName, &apiv1.VolumeDefinition{ + VolumeNumber: svd.VolumeNumber, + SizeKib: svd.SizeKib, + }) + if errors.Is(err, store.ErrAlreadyExists) { + err = volumeAlreadyRestored(ctx, run, rdName, svd) + } + + if err != nil { + return fmt.Errorf("restore volume %d onto %s: %w", svd.VolumeNumber, rdName, err) + } + } + + return nil +} + +// errVolumeGoneOnReadBack reports a volume the create said exists and the +// read-back did not find: it was deleted in between, which is not a taken +// number, and the restore can simply be run again. +var errVolumeGoneOnReadBack = errors.New("the create said it exists, but it was gone when read back") + +// volumeAlreadyRestored accepts an existing volume as the one the snapshot +// would have written when it is at least that volume's size: grown is the +// restore's own volume expanded since, the way store.AssessLeftover and the +// REST door judge it, while smaller is somebody else's. +func volumeAlreadyRestored(ctx context.Context, run *runContext, rdName string, svd *apiv1.SnapshotVolumeDef) error { + vds, err := run.Store.VolumeDefinitions().List(ctx, rdName) + if err != nil { + return fmt.Errorf("read back the existing volume: %w", err) + } + + for i := range vds { + if vds[i].VolumeNumber != svd.VolumeNumber { + continue + } + + if vds[i].SizeKib < svd.SizeKib { + return fmt.Errorf("%w: it holds %d KiB where the snapshot recorded %d KiB", + errVolumeNumberTaken, vds[i].SizeKib, svd.SizeKib) + } + + return nil + } + + return errVolumeGoneOnReadBack +} + // rollbackRestore removes the definition a failed restore had already // created, and returns the failure that caused it. // @@ -372,15 +720,150 @@ func snapshotRestoreResource(ctx context.Context, run *runContext) error { // A rollback that itself fails must not replace the original error: // the first one is what the operator needs, the second is a note about // what was left behind. -func rollbackRestore(ctx context.Context, run *runContext, rdName string, cause error) error { - err := run.Store.ResourceDefinitions().Delete(ctx, rdName) +// +// The definition this command created can be adopted by another request while +// it runs, which then finishes it and may already have answered for it. The +// rollback holds the same handshake the REST door does (store.HoldRollback) +// before it deletes anything, and leaves an adopted definition to its adopter. +// One that already reads as finished is left too: a re-run that found it so +// reported it restored without adopting it. +func rollbackRestore( + ctx context.Context, run *runContext, rdName string, planned []apiv1.Resource, cause error, +) error { + err := store.HoldRollback(ctx, run.Store, rdName, true) + if errors.Is(err, store.ErrRollbackGone) { + return cause + } + + if errors.Is(err, store.ErrRollbackYielded) { + return fmt.Errorf("%w (%s was not rolled back, because another request marked it adopted; "+ + "if that request did not finish it either, %s)", cause, rdName, restoreAgain(run)) + } + + if errors.Is(err, store.ErrRollbackAnswered) { + return answeredRestoreLeft(ctx, run, rdName, planned, cause) + } + + if err != nil { + return fmt.Errorf("%w (%s was not rolled back, since whether another request took it over "+ + "or reported it restored could not be checked: %w; %s to finish it)", + cause, rdName, err, restoreAgain(run)) + } + + err = store.EnterDestructiveRollback(ctx, run.Store, rdName, store.RollbackStepDeleteDefinition) + if errors.Is(err, store.ErrRollbackGone) { + return cause + } + + if err != nil { + return fmt.Errorf("%w (%s was not rolled back: %w; %s to finish it)", + cause, rdName, err, restoreAgain(run)) + } + + err = run.Store.ResourceDefinitions().Delete(ctx, rdName) if err != nil && !errors.Is(err, store.ErrNotFound) { - return fmt.Errorf("%w (rolling back %s also failed: %w)", cause, rdName, err) + // The definition stays, so the mark that says it is being rolled + // back must not: nothing is rolling it back now, and a retry that + // met the mark would be refused for good instead of finishing it. + releaseErr := store.ReleaseRollback(ctx, run.Store, rdName) + if releaseErr != nil { + return fmt.Errorf("%w (rolling back %s also failed: %w; it still carries the rollback mark, "+ + "so delete it by hand: %w)", cause, rdName, err, releaseErr) + } + + return fmt.Errorf("%w (rolling back %s also failed: %w; %s to finish it)", + cause, rdName, err, restoreAgain(run)) + } + + // A volume that went away under the read-back is a race, not a refusal; + // with the target rolled back, the same command starts it over. + if errors.Is(cause, errVolumeGoneOnReadBack) { + return fmt.Errorf("%w (%s was rolled back; run the same command again)", cause, rdName) } return cause } +// restoreAgain names the restore that finishes a definition the rollback left, +// as a command rather than "the same command again": `rd clone` runs this +// restore under the hood and refuses a target that already exists, so +// re-running the clone is a dead end, while the restore of its internal +// snapshot resumes it. The nodes and the pool the operator chose go with it: +// without them the resume places a replica on every node holding the snapshot, +// or finds no pool where the operator had to name one. +func restoreAgain(run *runContext) string { + cmd := "blockstor snapshot resource restore --from-resource " + run.Flags.Values["from-resource"] + + " --from-snapshot " + run.Flags.Values["from-snapshot"] + + " --to-resource " + run.Flags.Values["to-resource"] + + if len(run.Flags.Nodes) > 0 { + cmd += " --nodes " + strings.Join(run.Flags.Nodes, ",") + } + + if pool := run.Flags.Values["storage-pool"]; pool != "" { + cmd += " --storage-pool " + pool + } + + return "run `" + cmd + "`" +} + +// placeOverExisting places a restore's replicas onto a definition that was +// already there, prepared or this restore's own leftover. Nothing rolls such a +// placement back, and once one replica has landed a re-run judges the +// definition finished and places nothing, so a failure names the replicas +// that never landed and the command that places them. +func placeOverExisting(ctx context.Context, run *runContext, planned []apiv1.Resource) error { + err := stampRestoredReplicas(ctx, run, planned) + if err == nil || len(planned) == 0 { + return err + } + + rdName := planned[0].Name + + missing, missErr := store.MissingReplicas(ctx, run.Store, rdName, plannedNodes(planned)) + if missErr != nil || len(missing) == 0 || len(missing) == len(planned) { + return err + } + + return fmt.Errorf("%w (%s has no replica on %s, which a re-run will not place: "+ + "run `blockstor resource create %s` for each)", err, rdName, strings.Join(missing, ", "), rdName) +} + +// plannedNodes names the nodes a restore planned replicas on. +func plannedNodes(planned []apiv1.Resource) []string { + nodes := make([]string, 0, len(planned)) + for i := range planned { + nodes = append(nodes, planned[i].NodeName) + } + + return nodes +} + +// answeredRestoreLeft is the error for a definition the rollback left because +// it already reads as finished. A re-run judges it finished too and places +// nothing, so a replica this run planned and never placed is named with the +// command that places it, rather than left for a re-run that will not. +func answeredRestoreLeft( + ctx context.Context, run *runContext, rdName string, planned []apiv1.Resource, cause error, +) error { + missing, err := store.MissingReplicas(ctx, run.Store, rdName, plannedNodes(planned)) + if err != nil { + return fmt.Errorf("%w (%s was not rolled back: it already holds every volume and a replica, "+ + "and another run may have reported it restored; which of its replicas are missing "+ + "could not be read: %w)", cause, rdName, err) + } + + if len(missing) == 0 { + return fmt.Errorf("%w (%s was not rolled back: it already holds every volume and a replica, "+ + "and another run may have reported it restored)", cause, rdName) + } + + return fmt.Errorf("%w (%s was not rolled back: it already holds every volume and a replica, "+ + "and another run may have reported it restored; it has no replica on %s, which a re-run "+ + "will not place: run `blockstor resource create %s` for each)", + cause, rdName, strings.Join(missing, ", "), rdName) +} + // snapshotRestoreVolumeDefinition implements `snapshot // volume-definition restore`: the snapshot's recorded volume layout is // hydrated onto an EXISTING definition, which is created separately. @@ -395,11 +878,31 @@ func snapshotRestoreVolumeDefinition(ctx context.Context, run *runContext) error return fmt.Errorf("get snapshot %s of %s: %w", args.fromSnapshot, args.fromResource, err) } - _, err = run.Store.ResourceDefinitions().Get(ctx, args.toResource) + // Read past the cache, for the marker below. + target, err := run.Store.ResourceDefinitions().GetUncached(ctx, args.toResource) if err != nil { return fmt.Errorf("get resource definition %s: %w", args.toResource, err) } + // A definition being deleted is refused, as `s resource restore` refuses + // it through store.PreparedRestoreTarget: the volumes written here would + // be reaped as they land, and the command would report a restore that is + // not there. + if slices.Contains(target.Flags, apiv1.ResourceFlagDelete) { + return fmt.Errorf("%w: %s; wait for the delete to finish, then restore into a fresh definition", + errTargetBeingDeleted, args.toResource) + } + + // A definition carrying a restore marker is being restored or cloned by + // another operation, which hydrates its volumes itself and keeps one of + // its own snapshot's size. Writing them here as well lets this command + // fail on a later volume and unwind one that operation answered for. + // The marker goes on with the definition, so it is always seen. + if marker := target.Props[store.RestoreFromSnapshotProp]; marker != "" { + return fmt.Errorf("%w: %s carries %q; restore it with `s resource restore` instead", + errTargetBeingRestored, args.toResource, marker) + } + // The collision is refused BEFORE anything is written: hydrating // volume by volume would land the non-colliding ones first and // leave the target half-restored. @@ -448,6 +951,13 @@ func hydrateVolumes(ctx context.Context, run *runContext, rdName string, snap *a // definition carrying an arbitrary subset of the snapshot — // a shape nothing downstream expects. Take back exactly // what this call added. + // + // A volume a concurrent restore of the same snapshot wrote is a + // failure here too, not one to accept: both create in the + // snapshot's order, so the run that loses fails on the first + // volume with nothing added, and its unwind takes nothing the + // other run answered for. Accepting it lets the loser go on, + // fail later, and unwind volumes the winner is using. unwindVolumes(ctx, run, rdName, added) return fmt.Errorf("restore volume %d onto %s: %w", svd.VolumeNumber, rdName, err) @@ -473,14 +983,17 @@ func unwindVolumes(ctx context.Context, run *runContext, rdName string, added [] } } -// placeRestored stamps replicas of the restored definition. +// planRestoredReplicas decides the replicas a restore places, without +// writing anything. // // They land on the nodes that HOLD THE SNAPSHOT, in the same pool the // source replica uses there — never via the autoplacer. A restored // replica on a different backend makes the satellite pipe the // snapshot stream into the wrong receiver, which never converges. -// Explicit `--node-name` values win when the operator gave them. -func placeRestored(ctx context.Context, run *runContext, srcRD, rdName string, snap *apiv1.Snapshot) error { +// Explicit `--nodes` values win when the operator gave them. +func planRestoredReplicas( + ctx context.Context, run *runContext, srcRD, rdName string, snap *apiv1.Snapshot, +) ([]apiv1.Resource, error) { nodes := run.Flags.Nodes // Named nodes have to actually hold the snapshot. Restoring onto one @@ -490,7 +1003,7 @@ func placeRestored(ctx context.Context, run *runContext, srcRD, rdName string, s err := validate.RestoreNodesHoldSnapshot(nodes, snap.Nodes) if err != nil { //nolint:wrapcheck // a semantic refusal; see checkVolumeSize - return err + return nil, err } if len(nodes) == 0 { @@ -503,29 +1016,145 @@ func placeRestored(ctx context.Context, run *runContext, srcRD, rdName string, s // A Snapshot CR not written through this CLI can carry an empty // node list, so the restore side needs the matching guard. if len(nodes) == 0 { - return fmt.Errorf("%s names no node to restore %s onto: %w", + return nil, fmt.Errorf("%s names no node to restore %s onto: %w", snap.Name, rdName, errNothingToRestore) } + planned := make([]apiv1.Resource, 0, len(nodes)) + for _, node := range nodes { - res := &apiv1.Resource{Name: rdName, NodeName: node} + res := apiv1.Resource{Name: rdName, NodeName: node} pool, err := sourcePoolOn(ctx, run, srcRD, node) if err != nil { - return err + return nil, err } - stampProp(res, storPoolNameProp, pool) + stampProp(&res, storPoolNameProp, pool) + + planned = append(planned, res) + } + + return planned, nil +} - err = run.Store.Resources().Create(ctx, res) +// errReplicaStillDeleting refuses a restore whose replica on a node is +// still being torn down: counting it as placed would report a restore +// with no live replica there. +var errReplicaStillDeleting = errors.New("a replica of the restore target is still being deleted") + +// stampRestoredReplicas creates the planned replicas. One already on its +// node is the replica an earlier run of this restore placed and is taken +// as placed, unless it is being deleted. +func stampRestoredReplicas(ctx context.Context, run *runContext, planned []apiv1.Resource) error { + for i := range planned { + err := stampRestoredReplica(ctx, run, &planned[i]) if err != nil { - return fmt.Errorf("create restored replica %s on %s: %w", rdName, node, err) + return err + } + } + + return nil +} + +// stampRestoredReplica places one replica. An existing one is the restore's +// own unless it is being deleted or is an operator's diskless replica. A +// tie-breaker is the controller's witness and is promoted to the replica the +// restore wanted there; see promoteWitness. One that the create collided with and that +// is gone when read back was deleted in between, and is created again, the way +// the REST door does; a collision that keeps recurring is an error. +func stampRestoredReplica(ctx context.Context, run *runContext, res *apiv1.Resource) error { + for range replicaCollisionAttempts { + err := run.Store.Resources().Create(ctx, res) + if err == nil { + return nil + } + + if !errors.Is(err, store.ErrAlreadyExists) { + return fmt.Errorf("create restored replica %s on %s: %w", res.Name, res.NodeName, err) + } + + existing, getErr := store.GetResourceUncached(ctx, run.Store, res.Name, res.NodeName) + if errors.Is(getErr, store.ErrNotFound) { + continue + } + + if getErr != nil { + return fmt.Errorf("read replica %s on %s: %w", res.Name, res.NodeName, getErr) + } + + if slices.Contains(existing.Flags, apiv1.ResourceFlagDelete) { + return fmt.Errorf("%w: %s on %s", errReplicaStillDeleting, res.Name, res.NodeName) } + + if slices.Contains(existing.Flags, apiv1.ResourceFlagTieBreaker) { + return promoteWitness(ctx, run, res) + } + + if !store.HoldsData(&existing) { + return fmt.Errorf("%w: %s on %s; delete it, then retry", errReplicaHoldsNoData, res.Name, res.NodeName) + } + + return nil + } + + return fmt.Errorf("%w: %s on %s", errReplicaCollisionRecurs, res.Name, res.NodeName) +} + +// promoteWitness turns the controller's tie-breaker witness on a node the +// restore places onto into the replica it asked for, the way the REST door's +// autoplace promotes one. Its volumes are then restored from the snapshot like +// any other replica of a marked target. +func promoteWitness(ctx context.Context, run *runContext, res *apiv1.Resource) error { + err := run.Store.Resources().PatchResourceSpec(ctx, res.Name, res.NodeName, func(live *apiv1.Resource) error { + if slices.Contains(live.Flags, apiv1.ResourceFlagDelete) { + return fmt.Errorf("%w: %s on %s", errReplicaStillDeleting, res.Name, res.NodeName) + } + + if !slices.Contains(live.Flags, apiv1.ResourceFlagTieBreaker) { + return fmt.Errorf("%w: %s on %s is no longer the controller's witness", errReplicaHoldsNoData, res.Name, res.NodeName) + } + + live.Flags, _ = apiv1.PromoteWitnessFlags(live.Flags, true) + + if pool := res.Props["StorPoolName"]; pool != "" { + if live.Props == nil { + live.Props = map[string]string{} + } + + live.Props["StorPoolName"] = pool + } + + return nil + }) + if err != nil { + return fmt.Errorf("promote the witness of %s on %s: %w", res.Name, res.NodeName, err) } return nil } +// replicaCollisionAttempts bounds the create-then-read-back loop for a +// replica that keeps colliding with one that is gone when read back. +const replicaCollisionAttempts = 3 + +// errTargetBeingRestored refuses a volume-definition restore into a +// definition another restore or clone has marked as its own. +var errTargetBeingRestored = errors.New("the definition is the target of a restore or clone") + +// errTargetBeingDeleted refuses a volume-definition restore into a definition +// being deleted. +var errTargetBeingDeleted = errors.New("the definition is being deleted") + +// errReplicaHoldsNoData refuses a restore whose replica on a node already +// exists diskless: counted as placed, the restore reports success with no copy +// of the data there. +var errReplicaHoldsNoData = errors.New("a replica of the restore target is diskless and holds no copy of the data") + +// errReplicaCollisionRecurs reports a replica create that kept colliding with +// a replica nobody could read back. +var errReplicaCollisionRecurs = errors.New("the replica create kept colliding with a replica that was gone when read back") + // errNoSourcePool is returned when a restore cannot work out which // storage pool the new replica belongs in and the operator did not say. var errNoSourcePool = errors.New("no storage pool to restore into") diff --git a/internal/cli/snapshot_restore_doors_test.go b/internal/cli/snapshot_restore_doors_test.go new file mode 100644 index 00000000..c3bf92cd --- /dev/null +++ b/internal/cli/snapshot_restore_doors_test.go @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli_test + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/cozystack/blockstor/internal/cli" + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// `s vd restore` into a definition being deleted is refused before anything +// is written, as `s resource restore` refuses it: the volumes would be reaped +// as they land, and the command would report a restore that is not there. +func TestSnapshotVolumeDefinitionRestoreRefusesATargetBeingDeleted(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + seedSnapshotSource(ctx, backend) + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-dying", Flags: []string{apiv1.ResourceFlagDelete}, + }); err != nil { + t.Fatalf("seed the target: %v", err) + } + + var errBuf bytes.Buffer + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { return backend, nil }, + } + + if got := app.Run(ctx, []string{ + "s", "vd", "restore", "--from-resource", "pvc-x", "--from-snapshot", "snap-1", "--to-resource", "pvc-dying", + }); got == 0 { + t.Fatal("volume-definition restore into a definition being deleted succeeded") + } + + if vds, _ := backend.VolumeDefinitions().List(ctx, "pvc-dying"); len(vds) != 0 { + t.Errorf("the refused restore wrote %d volume(s)", len(vds)) + } + + if !strings.Contains(errBuf.String(), "being deleted") { + t.Errorf("stderr %q does not say the target is being deleted", errBuf.String()) + } +} + +// rollbackStartsUnderTheClaimRDs lands the creator's in-progress rollback mark +// just before the run's adoption patch: the creator began rolling back after +// the run judged the leftover finished and before it claimed it. +type rollbackStartsUnderTheClaimRDs struct { + store.ResourceDefinitionStore + + target string +} + +func (r rollbackStartsUnderTheClaimRDs) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + if name != r.target { + return r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) //nolint:wrapcheck // test double + } + + if err := r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, + func(rd *apiv1.ResourceDefinition) error { + rd.Props[store.RollbackAbandonedProp] = store.RollbackInProgress + + return nil + }); err != nil { + return err //nolint:wrapcheck // test double + } + + return r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) //nolint:wrapcheck // test double +} + +type rollbackStartsUnderTheClaimStore struct { + store.Store + + target string +} + +func (s rollbackStartsUnderTheClaimStore) ResourceDefinitions() store.ResourceDefinitionStore { + return rollbackStartsUnderTheClaimRDs{ResourceDefinitionStore: s.Store.ResourceDefinitions(), target: s.target} +} + +// A re-run over a finished leftover takes the adoption mark before it reports +// the restore done, as the REST replay does: the mark is what tells a creator +// still rolling back that somebody was answered for the definition. One that +// meets a creator already rolling back reports the restore as not done. +func TestSnapshotRestoreReRunOfAFinishedLeftoverTakesTheMark(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + for _, tc := range []struct { + name string + rollingUp bool + }{ + {name: "nobody rolling back"}, + {name: "the creator starts rolling back", rollingUp: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(ctx, backend) + seedLeftover(t, backend, "pvc-finished", "node-1") //nolint:contextcheck // the helper seeds on t's own context + + var st store.Store = backend + if tc.rollingUp { + st = rollbackStartsUnderTheClaimStore{Store: backend, target: "pvc-finished"} + } + + var outBuf, errBuf bytes.Buffer + + app := &cli.App{ + Out: &outBuf, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { return st, nil }, + } + + got := app.Run(ctx, restoreResourceArgv("pvc-finished")) + + rd, err := backend.ResourceDefinitions().Get(ctx, "pvc-finished") + if err != nil { + t.Fatalf("read the leftover: %v", err) + } + + if !tc.rollingUp { + if got != 0 || !strings.Contains(outBuf.String(), "already restored") { + t.Fatalf("re-run = %d, out %q, err %q: want it reported restored", got, outBuf.String(), errBuf.String()) + } + + if rd.Props[store.RestoreAdoptedProp] == "" { + t.Error("the re-run reported the restore done without taking the adoption mark") + } + + return + } + + if got == 0 || strings.Contains(outBuf.String(), "already restored") { + t.Errorf("re-run under a rollback = %d, out %q: want it refused", got, outBuf.String()) + } + }) + } +} + +// finishedUnderTheCreate plays a concurrent run that finished the restore +// between this run's read of the name and its create: the create collides +// with a finished leftover. With rollingBack, the creator of that leftover +// starts rolling it back inside this run's claim. +type finishedUnderTheCreate struct { + store.ResourceDefinitionStore + + t *testing.T + backend store.Store + rollingBack bool +} + +func (f finishedUnderTheCreate) Create(_ context.Context, rd *apiv1.ResourceDefinition) error { + seedLeftover(f.t, f.backend, rd.Name, "node-1") //nolint:contextcheck // the helper seeds on t's own context + + return store.ErrAlreadyExists +} + +func (f finishedUnderTheCreate) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + if !f.rollingBack { + return f.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) //nolint:wrapcheck // test double + } + + return rollbackStartsUnderTheClaimRDs{ResourceDefinitionStore: f.ResourceDefinitionStore, target: name}. + PatchResourceDefinitionSpec(ctx, name, mutate) +} + +type finishedUnderTheCreateStore struct { + store.Store + + t *testing.T + rollingBack bool +} + +func (s finishedUnderTheCreateStore) ResourceDefinitions() store.ResourceDefinitionStore { + return finishedUnderTheCreate{ + ResourceDefinitionStore: s.Store.ResourceDefinitions(), t: s.t, backend: s.Store, rollingBack: s.rollingBack, + } +} + +// The same holds when the finished leftover is met by a create that collided +// with it rather than by the read before it. +func TestSnapshotRestoreCollidingWithAFinishedRestoreTakesTheMark(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + rollingBack bool + }{ + {name: "nobody rolling back"}, + {name: "the creator starts rolling back", rollingBack: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + seedSnapshotSource(ctx, backend) + + var outBuf, errBuf bytes.Buffer + + app := &cli.App{ + Out: &outBuf, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { + return finishedUnderTheCreateStore{Store: backend, t: t, rollingBack: tc.rollingBack}, nil + }, + } + + got := app.Run(ctx, restoreResourceArgv("pvc-collided")) + + rd, err := backend.ResourceDefinitions().Get(ctx, "pvc-collided") + if err != nil { + t.Fatalf("read the leftover: %v", err) + } + + if !tc.rollingBack { + if got != 0 || !strings.Contains(outBuf.String(), "already restored") { + t.Fatalf("run = %d, out %q, err %q: want it reported restored", got, outBuf.String(), errBuf.String()) + } + + if rd.Props[store.RestoreAdoptedProp] == "" { + t.Error("the run reported the restore done without taking the adoption mark") + } + + return + } + + if got == 0 || strings.Contains(outBuf.String(), "already restored") { + t.Errorf("run under a rollback = %d, out %q: want it refused", got, outBuf.String()) + } + }) + } +} diff --git a/internal/cli/snapshot_restore_faults_test.go b/internal/cli/snapshot_restore_faults_test.go new file mode 100644 index 00000000..549edd0b --- /dev/null +++ b/internal/cli/snapshot_restore_faults_test.go @@ -0,0 +1,706 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli_test + +import ( + "bytes" + "context" + "errors" + "strings" + "sync/atomic" + "testing" + + "github.com/cozystack/blockstor/internal/cli" + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +var errInjected = errors.New("injected store failure") + +// restoreFaults names the calls a restoreFaultStore fails or bends for the +// restore target. A zero field passes the call through. +type restoreFaults struct { + // targetGet fails every read of the target definition. + targetGet error + // targetMissesOnce answers the first read of the target NotFound, the + // read a definition created a moment later did not see. + targetMissesOnce bool + // targetGetAfter fails reads of the target past the first one with + // this error: the read that found it succeeds, the next does not. + targetGetAfter error + // volumeCollides answers AlreadyExists to every volume create, and + // volumeList fails the read-back that follows. + volumeCollides bool + volumeList error + // rdCreate fails the definition create. + rdCreate error + // replicaCollides answers AlreadyExists to every replica create, and + // replicaGet or replicaDeleting decide what the read-back finds. + replicaCollides bool + replicaGet error + replicaDeleting bool + // replicaCreate fails every replica create, which sends a restore that + // created its definition into its rollback. + replicaCreate error + // replicaCreateOn narrows replicaCreate to the replica on this node, so + // the restore has placed the others when it fails. + replicaCreateOn string + // rdDelete fails the rollback's delete of the definition. + rdDelete error + // rdPatch fails every definition patch past the first patchesAllowed. + rdPatch error + patchesAllowed int32 +} + +type restoreFaultStore struct { + store.Store + + target string + f restoreFaults + missed *atomic.Bool + patches *atomic.Int32 + reads *atomic.Int32 +} + +func (s restoreFaultStore) ResourceDefinitions() store.ResourceDefinitionStore { + return restoreFaultRDs{ResourceDefinitionStore: s.Store.ResourceDefinitions(), s: s} +} + +func (s restoreFaultStore) VolumeDefinitions() store.VolumeDefinitionStore { + return restoreFaultVolumes{VolumeDefinitionStore: s.Store.VolumeDefinitions(), s: s} +} + +type restoreFaultVolumes struct { + store.VolumeDefinitionStore + + s restoreFaultStore +} + +func (v restoreFaultVolumes) Create(ctx context.Context, rdName string, vd *apiv1.VolumeDefinition) error { + if v.s.f.volumeCollides { + return store.ErrAlreadyExists + } + + return v.VolumeDefinitionStore.Create(ctx, rdName, vd) //nolint:wrapcheck // pass-through test double +} + +func (v restoreFaultVolumes) List(ctx context.Context, rdName string) ([]apiv1.VolumeDefinition, error) { + if v.s.f.volumeList != nil { + return nil, v.s.f.volumeList + } + + return v.VolumeDefinitionStore.List(ctx, rdName) //nolint:wrapcheck // pass-through test double +} + +func (s restoreFaultStore) Resources() store.ResourceStore { + return restoreFaultReplicas{ResourceStore: s.Store.Resources(), s: s} +} + +type restoreFaultRDs struct { + store.ResourceDefinitionStore + + s restoreFaultStore +} + +func (r restoreFaultRDs) Get(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + if name == r.s.target { + if r.s.f.targetGet != nil { + return apiv1.ResourceDefinition{}, r.s.f.targetGet + } + + if r.s.f.targetMissesOnce && r.s.missed.CompareAndSwap(false, true) { + return apiv1.ResourceDefinition{}, store.ErrNotFound + } + + if r.s.f.targetGetAfter != nil && r.s.reads.Add(1) > 1 { + return apiv1.ResourceDefinition{}, r.s.f.targetGetAfter + } + } + + return r.ResourceDefinitionStore.Get(ctx, name) //nolint:wrapcheck // pass-through test double +} + +func (r restoreFaultRDs) Delete(ctx context.Context, name string) error { + if r.s.f.rdDelete != nil { + return r.s.f.rdDelete + } + + return r.ResourceDefinitionStore.Delete(ctx, name) //nolint:wrapcheck // pass-through test double +} + +func (r restoreFaultRDs) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + if r.s.f.rdPatch != nil && r.s.patches.Add(1) > r.s.f.patchesAllowed { + return r.s.f.rdPatch + } + + return r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) //nolint:wrapcheck // pass-through test double +} + +func (r restoreFaultRDs) Create(ctx context.Context, rd *apiv1.ResourceDefinition) error { + if r.s.f.rdCreate != nil { + return r.s.f.rdCreate + } + + return r.ResourceDefinitionStore.Create(ctx, rd) //nolint:wrapcheck // pass-through test double +} + +type restoreFaultReplicas struct { + store.ResourceStore + + s restoreFaultStore +} + +func (r restoreFaultReplicas) Create(ctx context.Context, res *apiv1.Resource) error { + if r.s.f.replicaCreate != nil && (r.s.f.replicaCreateOn == "" || r.s.f.replicaCreateOn == res.NodeName) { + return r.s.f.replicaCreate + } + + if r.s.f.replicaCollides { + return store.ErrAlreadyExists + } + + return r.ResourceStore.Create(ctx, res) //nolint:wrapcheck // pass-through test double +} + +func (r restoreFaultReplicas) Get(ctx context.Context, rdName, node string) (apiv1.Resource, error) { + if r.s.f.replicaGet != nil { + return apiv1.Resource{}, r.s.f.replicaGet + } + + if r.s.f.replicaDeleting { + return apiv1.Resource{Name: rdName, NodeName: node, Flags: []string{apiv1.ResourceFlagDelete}}, nil + } + + return r.ResourceStore.Get(ctx, rdName, node) //nolint:wrapcheck // pass-through test double +} + +func runRestoreWithFaults(t *testing.T, backend store.Store, target string, f restoreFaults, extra ...string) (int, string) { + t.Helper() + + var ( + errBuf bytes.Buffer + missed atomic.Bool + patches atomic.Int32 + reads atomic.Int32 + ) + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { + return restoreFaultStore{Store: backend, target: target, f: f, missed: &missed, patches: &patches, reads: &reads}, nil + }, + } + + argv := restoreResourceArgv(target) + if len(extra) > 0 { + argv = append([]string{ + "s", "resource", "restore", "--from-resource", "pvc-x", "--from-snapshot", extra[0], "--to-resource", target, + }, extra[1:]...) + } + + return app.Run(t.Context(), argv), errBuf.String() +} + +// A restore that cannot read, create or place its target stops with the error +// and leaves nothing behind: no definition it made, no replica. The one +// exception is a failure that also blinds the rollback's own handshake: one +// that cannot read the volumes cannot tell whether another run already reported +// the definition restored, so it leaves the definition, says so, and the same +// restore run again finishes it. +func TestSnapshotRestoreStopsOnAFailedStoreCall(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + f restoreFaults + stderr string + kept bool + }{ + {name: "target-read", f: restoreFaults{targetGet: errInjected}, stderr: "injected store failure"}, + {name: "definition-create", f: restoreFaults{rdCreate: errInjected}, stderr: "injected store failure"}, + { + name: "replica-read-back", f: restoreFaults{replicaCollides: true, replicaGet: errInjected}, + stderr: "injected store failure", + }, + { + name: "replica-still-deleting", f: restoreFaults{replicaCollides: true, replicaDeleting: true}, + stderr: "still being deleted", + }, + { + name: "volume-read-back", f: restoreFaults{volumeCollides: true, volumeList: errInjected}, + stderr: "was not rolled back", kept: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + got, stderr := runRestoreWithFaults(t, backend, "pvc-fault", tc.f) + if got == 0 { + t.Fatal("the restore succeeded over a failed store call") + } + + if !strings.Contains(stderr, tc.stderr) { + t.Errorf("stderr %q does not carry %q", stderr, tc.stderr) + } + + if tc.kept { + assertRestoreLeftUnmarked(t, backend, "pvc-fault") + + return + } + + if _, err := backend.ResourceDefinitions().Get(t.Context(), "pvc-fault"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the failed restore left its definition behind: %v", err) + } + + replicas, err := backend.Resources().ListByDefinition(t.Context(), "pvc-fault") + if err != nil || len(replicas) != 0 { + t.Errorf("the failed restore left replicas %v (%v)", replicas, err) + } + }) + } +} + +// A definition that appears between the read that found none and the create +// is the target, judged like any other existing one: here one prepared for +// this restore, so it is taken, not refused and not left half-done. +func TestSnapshotRestoreJudgesATargetThatAppearedBeforeTheCreate(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + var errBuf bytes.Buffer + + prepareTarget(t, &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { + return backend, nil + }, + }, &errBuf, "pvc-late") + + got, stderr := runRestoreWithFaults(t, backend, "pvc-late", restoreFaults{targetMissesOnce: true}) + if got != 0 { + t.Fatalf("exit = %d (stderr: %s), want the late target taken", got, stderr) + } + + assertRestoredInto(t, backend, "pvc-late", "node-1", "node-2") +} + +// A snapshot that captured no volumes restores nothing, and saying it restored +// would hand back a definition with no data. The definition the restore made +// for it is taken back. +func TestSnapshotRestoreRefusesASnapshotWithNoVolumes(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + if err := backend.Snapshots().Create(t.Context(), &apiv1.Snapshot{ + Name: "snap-empty", ResourceName: "pvc-x", Nodes: []string{"node-1"}, + }); err != nil { + t.Fatalf("seed the empty snapshot: %v", err) + } + + got, stderr := runRestoreWithFaults(t, backend, "pvc-empty", restoreFaults{}, "snap-empty") + if got == 0 { + t.Fatal("restoring a snapshot that captured no volumes succeeded") + } + + if !strings.Contains(stderr, "captured no volumes") { + t.Errorf("stderr %q does not say the snapshot captured nothing", stderr) + } + + if _, err := backend.ResourceDefinitions().Get(t.Context(), "pvc-empty"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the definition made for nothing was not taken back: %v", err) + } +} + +// A rollback that cannot hold the handshake deletes nothing: it cannot tell +// whether another request took the definition over and answered for it. +func TestSnapshotRestoreRollbackDeletesNothingWhenTheHandshakeFails(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + got, stderr := runRestoreWithFaults(t, backend, "pvc-nohold", + restoreFaults{replicaCreate: errInjected, rdPatch: errInjected}) + if got == 0 { + t.Fatal("the restore whose placement failed succeeded") + } + + if !strings.Contains(stderr, "could not be checked") { + t.Errorf("stderr %q does not say the handshake could not be checked", stderr) + } + + if _, err := backend.ResourceDefinitions().Get(t.Context(), "pvc-nohold"); err != nil { + t.Errorf("the rollback deleted a definition it could not check: %v", err) + } +} + +// A rollback that held the handshake, could not delete, and then could not +// take its mark back off says the definition still carries it, so the +// operator knows a retry would meet it. +func TestSnapshotRestoreRollbackSaysWhenItsMarkStays(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + got, stderr := runRestoreWithFaults(t, backend, "pvc-markstays", restoreFaults{ + // The hold and the step mark land; the release does not. + replicaCreate: errInjected, rdDelete: errInjected, rdPatch: errInjected, patchesAllowed: 2, + }) + if got == 0 { + t.Fatal("the restore whose placement failed succeeded") + } + + if !strings.Contains(stderr, "still carries the rollback mark") { + t.Errorf("stderr %q does not say the mark stayed", stderr) + } + + rd, err := backend.ResourceDefinitions().Get(t.Context(), "pvc-markstays") + if err != nil { + t.Fatalf("read the definition: %v", err) + } + + if rd.Props[store.RollbackAbandonedProp] != store.RollbackStepDeleteDefinition { + t.Errorf("fixture: rollback mark = %q, want the one that could not be cleared", rd.Props[store.RollbackAbandonedProp]) + } +} + +// seedLeftover puts an earlier attempt's leftover of this restore under the +// name: the marker and the snapshot's volume, plus the given replicas. +func seedLeftover(t *testing.T, backend store.Store, target string, nodes ...string) { + t.Helper() + + ctx := t.Context() + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: target, ResourceGroupName: "grp", + Props: map[string]string{store.RestoreFromSnapshotProp: "pvc-x:snap-1"}, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + if err := backend.VolumeDefinitions().Create(ctx, target, + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 1 << 20}); err != nil { + t.Fatalf("seed the leftover's volume: %v", err) + } + + for _, node := range nodes { + if err := backend.Resources().Create(ctx, &apiv1.Resource{Name: target, NodeName: node}); err != nil { + t.Fatalf("seed the leftover's replica: %v", err) + } + } +} + +// A resume that cannot claim the leftover writes nothing onto it: without the +// claim, the attempt that created it could roll it back under this run. +func TestSnapshotRestoreResumeWritesNothingWithoutItsClaim(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + seedLeftover(t, backend, "pvc-noclaim") + + got, stderr := runRestoreWithFaults(t, backend, "pvc-noclaim", restoreFaults{rdPatch: errInjected}) + if got == 0 { + t.Fatal("the resume went on without its claim") + } + + if !strings.Contains(stderr, "injected store failure") { + t.Errorf("stderr %q does not carry the claim's failure", stderr) + } + + replicas, err := backend.Resources().ListByDefinition(t.Context(), "pvc-noclaim") + if err != nil || len(replicas) != 0 { + t.Errorf("the unclaimed resume placed replicas %v (%v)", replicas, err) + } +} + +var errOutputClosed = errors.New("output closed") + +// failingWriter is a stdout that cannot be written. +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errOutputClosed } + +// "Already restored" is the whole answer of a re-run over a finished restore. +// One that could not be printed is a failed command, not a silent success. +func TestSnapshotRestoreRerunFailsWhenItCannotSayItIsDone(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + seedLeftover(t, backend, "pvc-done-quiet", "node-1") + + var errBuf bytes.Buffer + + app := &cli.App{ + Out: failingWriter{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { + return backend, nil + }, + } + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-done-quiet")); got == 0 { + t.Fatal("the re-run exited 0 without saying the restore is done") + } + + if !strings.Contains(errBuf.String(), "write output") { + t.Errorf("stderr %q does not say the output failed", errBuf.String()) + } +} + +// The read that judges an existing target is not the read that found it, and +// a failure there stops the restore before anything is written onto it. +func TestSnapshotRestoreStopsWhenTheExistingTargetCannotBeRead(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + seedLeftover(t, backend, "pvc-unread") + + got, stderr := runRestoreWithFaults(t, backend, "pvc-unread", restoreFaults{targetGetAfter: errInjected}) + if got == 0 { + t.Fatal("the restore went on over a target it could not read") + } + + if !strings.Contains(stderr, "injected store failure") { + t.Errorf("stderr %q does not carry the read's failure", stderr) + } + + replicas, err := backend.Resources().ListByDefinition(t.Context(), "pvc-unread") + if err != nil || len(replicas) != 0 { + t.Errorf("the restore placed replicas %v (%v) over a target it could not read", replicas, err) + } +} + +// assertRestoreLeftUnmarked checks a definition the rollback could not judge +// is still there and carries no rollback mark, so the same restore run again +// resumes it rather than being refused. +func assertRestoreLeftUnmarked(t *testing.T, backend store.Store, rdName string) { + t.Helper() + + rd, err := backend.ResourceDefinitions().Get(t.Context(), rdName) + if err != nil { + t.Fatalf("the definition the rollback could not judge is gone: %v", err) + } + + if mark := rd.Props[store.RollbackAbandonedProp]; mark != "" { + t.Errorf("the definition left in place carries rollback mark %q", mark) + } +} + +// A restore whose second replica fails has placed the first, so it already +// reads as finished, and another run may have reported it restored: the +// rollback leaves it. A re-run judges it finished too and places nothing, so +// the error names the node that never got its replica and how to place it. +func TestSnapshotRestoreLeftShortNamesTheReplicaItNeverPlaced(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + got, stderr := runRestoreWithFaults(t, backend, "pvc-short", + restoreFaults{replicaCreate: errInjected, replicaCreateOn: "node-2"}) + if got == 0 { + t.Fatal("the restore whose node-2 replica failed succeeded") + } + + if !strings.Contains(stderr, "no replica on node-2") || !strings.Contains(stderr, "blockstor resource create") { + t.Errorf("stderr %q does not name the replica it never placed", stderr) + } + + assertRestoreLeftUnmarked(t, backend, "pvc-short") +} + +// `rd clone` runs a restore of its internal snapshot, so a rollback that left +// the target told the operator to "run the same restore again", and re-running +// the clone is refused over a target that already exists. The advice names the +// restore command that resumes it, and following it finishes the clone. +func TestCloneLeftByItsRollbackNamesTheRestoreThatFinishesIt(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + var ( + errBuf bytes.Buffer + missed atomic.Bool + patches atomic.Int32 + reads atomic.Int32 + ) + + faulty := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { + return restoreFaultStore{ + Store: backend, target: "pvc-clone", f: restoreFaults{replicaCreate: errInjected, rdPatch: errInjected}, + missed: &missed, patches: &patches, reads: &reads, + }, nil + }, + } + + if got := faulty.Run(t.Context(), []string{"rd", "clone", "pvc-x", "pvc-clone"}); got == 0 { + t.Fatal("the clone whose placement failed succeeded") + } + + const resume = "blockstor snapshot resource restore --from-resource pvc-x --from-snapshot clone-pvc-clone " + + "--to-resource pvc-clone" + if !strings.Contains(errBuf.String(), resume) { + t.Fatalf("stderr %q does not name the restore that finishes the clone", errBuf.String()) + } + + clean := &cli.App{ + Out: &bytes.Buffer{}, + Err: &bytes.Buffer{}, + StoreFor: func(context.Context) (store.Store, error) { return backend, nil }, + } + + argv := strings.Fields(strings.TrimPrefix(resume, "blockstor ")) + argv[0], argv[1], argv[2] = "s", "resource", "restore" + + if got := clean.Run(t.Context(), argv); got != 0 { + t.Fatalf("following the advice = exit %d, want 0", got) + } + + replicas, err := backend.Resources().ListByDefinition(t.Context(), "pvc-clone") + if err != nil || len(replicas) == 0 { + t.Errorf("following the advice left the clone with replicas %v (%v)", replicas, err) + } +} + +// The advice a rollback leaves carries the nodes and the pool the operator +// named: without them the restore it names places a replica on every node +// holding the snapshot, a node the operator deliberately left out included. +func TestTheAdviceToRestoreAgainKeepsTheNodesAndPoolTheOperatorNamed(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + got, stderr := runRestoreWithFaults(t, backend, "pvc-onenode", + restoreFaults{replicaCreate: errInjected, rdPatch: errInjected}, + "snap-1", "--nodes", "node-1", "--storage-pool", "pool-1") + if got == 0 { + t.Fatal("the restore whose placement failed succeeded") + } + + const resume = "blockstor snapshot resource restore --from-resource pvc-x --from-snapshot snap-1 " + + "--to-resource pvc-onenode --nodes node-1 --storage-pool pool-1" + if !strings.Contains(stderr, resume) { + t.Fatalf("stderr %q does not name the restore with the operator's nodes and pool", stderr) + } + + clean := &cli.App{ + Out: &bytes.Buffer{}, + Err: &bytes.Buffer{}, + StoreFor: func(context.Context) (store.Store, error) { return backend, nil }, + } + + argv := strings.Fields(strings.TrimPrefix(resume, "blockstor ")) + argv[0], argv[1], argv[2] = "s", "resource", "restore" + + if code := clean.Run(t.Context(), argv); code != 0 { + t.Fatalf("following the advice = exit %d, want 0", code) + } + + replicas, err := backend.Resources().ListByDefinition(t.Context(), "pvc-onenode") + if err != nil || len(replicas) != 1 || replicas[0].NodeName != "node-1" { + t.Errorf("following the advice placed %v (%v), want node-1 only", replicas, err) + } +} + +// A snapshot that recorded no volumes has nothing to restore. A resume over a +// leftover of one hydrated nothing, placed a replica and exited 0 over a +// definition with no volume, and a fresh restore wrote the definition before +// refusing. Both are refused before anything is written. +func TestSnapshotRestoreOfAnEmptySnapshotWritesNothing(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + leftover bool + }{ + {name: "fresh"}, + {name: "over-a-leftover", leftover: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + seedSnapshotSource(ctx, backend) + + empty := apiv1.Snapshot{Name: "snap-e", ResourceName: "pvc-x", Nodes: []string{"node-1"}} + if err := backend.Snapshots().Create(ctx, &empty); err != nil { + t.Fatalf("seed the empty snapshot: %v", err) + } + + if tc.leftover { + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-e", Props: store.WithRestoreMarker(nil, &empty), + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + } + + got, stderr := runRestoreWithFaults(t, backend, "dst-e", restoreFaults{}, "snap-e") + if got == 0 { + t.Fatal("the restore of an empty snapshot succeeded") + } + + if !strings.Contains(stderr, "captured no volumes") { + t.Errorf("stderr %q does not say the snapshot has nothing to restore", stderr) + } + + if replicas, _ := backend.Resources().ListByDefinition(ctx, "dst-e"); len(replicas) != 0 { + t.Errorf("the refused restore placed %d replica(s)", len(replicas)) + } + + _, err := backend.ResourceDefinitions().Get(ctx, "dst-e") + if !tc.leftover && !errors.Is(err, store.ErrNotFound) { + t.Errorf("the refused restore wrote the definition: %v", err) + } + }) + } +} + +// A restore into a prepared target, or the resume of its own leftover, has no +// rollback, and once one replica landed a re-run judges it finished and places +// nothing. So a placement that fails after some replicas landed names the +// ones that did not and how to place them, as the creating path does. +func TestSnapshotRestoreIntoAnExistingTargetNamesTheReplicaItNeverPlaced(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + var prepErr bytes.Buffer + + prepareTarget(t, &cli.App{ + Out: &bytes.Buffer{}, Err: &prepErr, + StoreFor: func(context.Context) (store.Store, error) { return backend, nil }, + }, &prepErr, "pvc-prep") + + got, stderr := runRestoreWithFaults(t, backend, "pvc-prep", + restoreFaults{replicaCreate: errInjected, replicaCreateOn: "node-2"}) + if got == 0 { + t.Fatal("the restore whose node-2 replica failed succeeded") + } + + if !strings.Contains(stderr, "no replica on node-2") || !strings.Contains(stderr, "blockstor resource create") { + t.Errorf("stderr %q does not name the replica it never placed", stderr) + } +} diff --git a/internal/cli/snapshot_restore_marker_test.go b/internal/cli/snapshot_restore_marker_test.go new file mode 100644 index 00000000..0ba97380 --- /dev/null +++ b/internal/cli/snapshot_restore_marker_test.go @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli_test + +import ( + "context" + "strings" + "testing" + + "github.com/cozystack/blockstor/internal/cli" + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +type foldingSnapshots struct{ store.SnapshotStore } + +func (f foldingSnapshots) Get(ctx context.Context, rdName, snapName string) (apiv1.Snapshot, error) { + return f.SnapshotStore.Get(ctx, strings.ToLower(rdName), strings.ToLower(snapName)) //nolint:wrapcheck // pass-through +} + +type foldingSnapshotStore struct{ store.Store } + +func (f foldingSnapshotStore) Snapshots() store.SnapshotStore { + return foldingSnapshots{f.Store.Snapshots()} +} + +// The marker is a store key for the placer and for the REST resume, so both +// halves are the stored spelling, not what the operator typed. The store folds +// snapshot names the way the Kubernetes store does, which is the only way the +// two spellings can differ and both be read. +func TestSnapshotResourceRestoreMarkerTakesTheStoredSpelling(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + folding := foldingSnapshotStore{backend} + + var out, errBuf strings.Builder + + app := &cli.App{ + Out: &out, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { + return folding, nil + }, + } + + argv := []string{ + "s", "resource", "restore", + "--from-resource", "pvc-x", "--from-snapshot", "SNAP-1", "--to-resource", "pvc-cased", + } + + if got := app.Run(t.Context(), argv); got != 0 { + t.Fatalf("restore exit = %d (stderr: %s)", got, errBuf.String()) + } + + def, err := backend.ResourceDefinitions().Get(t.Context(), "pvc-cased") + if err != nil { + t.Fatalf("get restored definition: %v", err) + } + + if got := def.Props["BlockstorRestoreFromSnapshot"]; got != "pvc-x:snap-1" { + t.Errorf("restore marker = %q, want pvc-x:snap-1, the spelling the snapshot is stored under", got) + } +} diff --git a/internal/cli/snapshot_restore_prepared_test.go b/internal/cli/snapshot_restore_prepared_test.go new file mode 100644 index 00000000..0a13771d --- /dev/null +++ b/internal/cli/snapshot_restore_prepared_test.go @@ -0,0 +1,1571 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli_test + +import ( + "bytes" + "context" + "errors" + "maps" + "slices" + "strings" + "sync/atomic" + "testing" + + "github.com/cozystack/blockstor/internal/cli" + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// LINSTOR documents a restore as `rd c`, `s vd restore`, then `s rsc +// restore`, and the last step refused the definition the first one made. +func TestSnapshotRestoreFinishesTheTargetLinstorsSequencePrepares(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + }) + + for _, argv := range [][]string{ + {"rd", "c", "pvc-prep"}, + {"s", "vd", "restore", "--from-resource", "pvc-x", "--from-snapshot", "snap-1", "--to-resource", "pvc-prep"}, + {"s", "resource", "restore", "--from-resource", "pvc-x", "--from-snapshot", "snap-1", "--to-resource", "pvc-prep"}, + } { + if got := app.Run(t.Context(), argv); got != 0 { + t.Fatalf("%v exit = %d (stderr: %s)", argv, got, errBuf.String()) + } + } + + rd, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-prep") + if err != nil { + t.Fatalf("read the restored definition: %v", err) + } + + if got := rd.Props[store.RestoreFromSnapshotProp]; got != "pvc-x:snap-1" { + t.Errorf("marker = %q, want %q", got, "pvc-x:snap-1") + } + + replicas, err := appStore(t, app).Resources().ListByDefinition(t.Context(), "pvc-prep") + if err != nil || len(replicas) == 0 { + t.Errorf("replicas after the restore = %d (%v), want the snapshot's nodes", len(replicas), err) + } +} + +// A definition that already has a live replica may hold data, and the +// restore into it stays refused. +func TestSnapshotRestoreRefusesATargetWithALiveReplica(t *testing.T) { + t.Parallel() + + app, _, _ := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + _ = backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "pvc-live"}) + _ = backend.VolumeDefinitions().Create(ctx, "pvc-live", &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 1 << 20}) + _ = backend.Resources().Create(ctx, &apiv1.Resource{Name: "pvc-live", NodeName: "node-1"}) + }) + + if got := app.Run(t.Context(), []string{ + "s", "resource", "restore", "--from-resource", "pvc-x", "--from-snapshot", "snap-1", "--to-resource", "pvc-live", + }); got == 0 { + t.Fatal("restore into a definition with a live replica succeeded") + } + + rd, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-live") + if err != nil { + t.Fatalf("read the target: %v", err) + } + + if marker := rd.Props[store.RestoreFromSnapshotProp]; marker != "" { + t.Errorf("the refused target was marked %q", marker) + } +} + +// A prepared target of another layer stack would hold the source's bytes +// under layers that did not write them, so the CLI refuses it the way the REST +// door does, in both directions, and leaves the operator's definition unmarked. +func TestSnapshotRestoreRefusesAPreparedTargetOfAnotherLayerStack(t *testing.T) { + t.Parallel() + + luks := []string{"DRBD", "LUKS", "STORAGE"} + + for _, tc := range []struct { + name string + source, target []string + }{ + {name: "plaintext-source-into-luks", source: []string{"DRBD", "STORAGE"}, target: luks}, + {name: "luks-source-into-plaintext", source: luks}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + _ = backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-src", LayerStack: tc.source, + }) + _ = backend.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "snap-1", ResourceName: "pvc-src", Nodes: []string{"node-1"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 1 << 20}}, + }) + _ = backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-dst", LayerStack: tc.target, + }) + _ = backend.VolumeDefinitions().Create(ctx, "pvc-dst", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 1 << 20}) + }) + + if got := app.Run(t.Context(), []string{ + "s", "resource", "restore", "--from-resource", "pvc-src", "--from-snapshot", "snap-1", + "--to-resource", "pvc-dst", "--storage-pool", "pool-1", + }); got == 0 { + t.Fatal("restore into a target of another layer stack succeeded") + } + + if !strings.Contains(errBuf.String(), "layer stack") { + t.Errorf("refusal %q does not name the layer stack", errBuf.String()) + } + + rd, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-dst") + if err != nil { + t.Fatalf("read the target: %v", err) + } + + if marker := rd.Props[store.RestoreFromSnapshotProp]; marker != "" { + t.Errorf("the refused target was marked %q", marker) + } + }) + } +} + +// prepareTarget runs the first two steps of LINSTOR's restore sequence. +func prepareTarget(t *testing.T, app *cli.App, errBuf *bytes.Buffer, target string) { + t.Helper() + + for _, argv := range [][]string{ + {"rd", "c", target}, + {"s", "vd", "restore", "--from-resource", "pvc-x", "--from-snapshot", "snap-1", "--to-resource", target}, + } { + if got := app.Run(t.Context(), argv); got != 0 { + t.Fatalf("%v exit = %d (stderr: %s)", argv, got, errBuf.String()) + } + } +} + +func restoreResourceArgv(target string, extra ...string) []string { + return append([]string{ + "s", "resource", "restore", "--from-resource", "pvc-x", "--from-snapshot", "snap-1", "--to-resource", target, + }, extra...) +} + +func assertRestoredInto(t *testing.T, st store.Store, target string, nodes ...string) { + t.Helper() + + rd, err := st.ResourceDefinitions().Get(t.Context(), target) + if err != nil { + t.Fatalf("read the restored definition: %v", err) + } + + if got := rd.Props[store.RestoreFromSnapshotProp]; got != "pvc-x:snap-1" { + t.Errorf("marker = %q, want %q", got, "pvc-x:snap-1") + } + + replicas, err := st.Resources().ListByDefinition(t.Context(), target) + if err != nil { + t.Fatalf("list the restored replicas: %v", err) + } + + got := make([]string, 0, len(replicas)) + for i := range replicas { + got = append(got, replicas[i].NodeName) + } + + slices.Sort(got) + + if !slices.Equal(got, nodes) { + t.Errorf("replicas on %v, want %v", got, nodes) + } +} + +// A request the restore refuses for its own shape, a node that does not hold +// the snapshot, left the marker on the operator's definition, and the +// corrected command then met "already exists" with no way out but deleting it. +func TestSnapshotRestoreIntoAPreparedTargetRefusesABadNodeBeforeMarkingIt(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, seedSnapshotSource) + prepareTarget(t, app, errBuf, "pvc-prep") + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-prep", "--nodes", "node-9")); got == 0 { + t.Fatal("restore onto a node without the snapshot succeeded") + } + + rd, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-prep") + if err != nil { + t.Fatalf("read the target: %v", err) + } + + if marker := rd.Props[store.RestoreFromSnapshotProp]; marker != "" { + t.Errorf("the refused request marked the target %q", marker) + } + + errBuf.Reset() + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-prep", "--nodes", "node-1")); got != 0 { + t.Fatalf("the corrected restore exit = %d (stderr: %s)", got, errBuf.String()) + } + + assertRestoredInto(t, appStore(t, app), "pvc-prep", "node-1") +} + +// errPlacementRefused is what the API server answers the second replica with. +var errPlacementRefused = errors.New("admission webhook refused the replica") + +// failOneReplica fails the first create of one replica, the way an API server +// that refuses one write does. +type failOneReplica struct { + store.ResourceStore + + node string + failed *atomic.Bool +} + +func (f failOneReplica) Create(ctx context.Context, res *apiv1.Resource) error { + if res.NodeName == f.node && f.failed.CompareAndSwap(false, true) { + return errPlacementRefused + } + + return f.ResourceStore.Create(ctx, res) //nolint:wrapcheck // pass-through test double +} + +type failOneReplicaStore struct { + store.Store + + node string + failed *atomic.Bool +} + +func (f failOneReplicaStore) Resources() store.ResourceStore { + return failOneReplica{ResourceStore: f.Store.Resources(), node: f.node, failed: f.failed} +} + +// A placement that failed after the marker went on left a definition no retry +// could finish: no longer prepared since it is marked, and refused by the +// create since it exists. Running the same command again finishes it. +func TestSnapshotRestoreIntoAPreparedTargetFinishesOnRetryAfterAPlacementFailure(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + var ( + errBuf bytes.Buffer + failed atomic.Bool + ) + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { + return failOneReplicaStore{Store: backend, node: "node-2", failed: &failed}, nil + }, + } + + prepareTarget(t, app, &errBuf, "pvc-prep") + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-prep")); got == 0 { + t.Fatal("the restore whose second replica was refused succeeded") + } + + errBuf.Reset() + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-prep")); got != 0 { + t.Fatalf("the retry exit = %d (stderr: %s)", got, errBuf.String()) + } + + // One live replica holds the restore, so the retry leaves placement + // where it is: topping it up is the placement reconciliation's job, the + // rule the REST replay follows too. + assertRestoredInto(t, backend, "pvc-prep", "node-1") +} + +// A retry over this restore's leftover whose every replica is going away +// refuses rather than placing over the tear-down. +func TestSnapshotRestoreRetryRefusesALeftoverReplicaBeingDeleted(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + _ = backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-left", Props: map[string]string{store.RestoreFromSnapshotProp: "pvc-x:snap-1"}, + }) + _ = backend.VolumeDefinitions().Create(ctx, "pvc-left", &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 1 << 20}) + _ = backend.Resources().Create(ctx, &apiv1.Resource{ + Name: "pvc-left", NodeName: "node-2", Flags: []string{apiv1.ResourceFlagDelete}, + }) + }) + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-left")); got == 0 { + t.Fatal("a retry placed over a leftover whose every replica is being deleted") + } + + if !strings.Contains(errBuf.String(), "being deleted") { + t.Errorf("refusal %q does not name the replicas being deleted", errBuf.String()) + } +} + +// Re-running a finished restore left a node the operator had emptied since +// alone on the REST door and re-stamped it here, restored from the +// point-in-time beside a replica that had moved on with live writes. +func TestSnapshotRestoreRerunLeavesAnEmptiedNodeAlone(t *testing.T) { + t.Parallel() + + app, outBuf, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + }) + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-done")); got != 0 { + t.Fatalf("restore exit = %d (stderr: %s)", got, errBuf.String()) + } + + if strings.Contains(outBuf.String(), "already restored") { + t.Errorf("the first restore said it was already done: %q", outBuf.String()) + } + + st := appStore(t, app) + assertRestoredInto(t, st, "pvc-done", "node-1", "node-2") + + if err := st.Resources().Delete(t.Context(), "pvc-done", "node-2"); err != nil { + t.Fatalf("empty node-2: %v", err) + } + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-done")); got != 0 { + t.Fatalf("re-run exit = %d (stderr: %s)", got, errBuf.String()) + } + + assertRestoredInto(t, st, "pvc-done", "node-1") + + // A re-run that exits 0 in silence reads as a restore that just ran. + if !strings.Contains(outBuf.String(), "pvc-done is already restored") { + t.Errorf("the re-run of a finished restore said nothing: %q", outBuf.String()) + } +} + +// A leftover an earlier attempt left when its rollback gave up is refused, +// the way the REST door refuses it, not finished. +func TestSnapshotRestoreRetryRefusesALeftoverWhoseRollbackGaveUp(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + _ = backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-gaveup", Props: map[string]string{ + store.RestoreFromSnapshotProp: "pvc-x:snap-1", + store.RollbackAbandonedProp: "replicas", + }, + }) + _ = backend.VolumeDefinitions().Create(ctx, "pvc-gaveup", &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 1 << 20}) + }) + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-gaveup")); got == 0 { + t.Fatal("a retry finished a leftover whose rollback gave up") + } + + if !strings.Contains(errBuf.String(), "rollback gave up") { + t.Errorf("refusal %q does not say the rollback gave up", errBuf.String()) + } + + replicas, err := appStore(t, app).Resources().ListByDefinition(t.Context(), "pvc-gaveup") + if err != nil || len(replicas) != 0 { + t.Errorf("the refused retry placed %d replica(s) (%v)", len(replicas), err) + } +} + +// A finished restore is judged before anything about the source is planned: +// the source losing the replicas its pools were read from failed a re-run +// over a target that was already done. +func TestSnapshotRestoreRerunOfAFinishedRestoreDoesNotNeedTheSource(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + }) + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-whole")); got != 0 { + t.Fatalf("restore exit = %d (stderr: %s)", got, errBuf.String()) + } + + st := appStore(t, app) + for _, node := range []string{"node-1", "node-2"} { + if err := st.Resources().Delete(t.Context(), "pvc-x", node); err != nil { + t.Fatalf("drop the source replica on %s: %v", node, err) + } + } + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-whole")); got != 0 { + t.Fatalf("re-run over a finished restore exit = %d (stderr: %s)", got, errBuf.String()) + } + + assertRestoredInto(t, st, "pvc-whole", "node-1", "node-2") +} + +// A prepared definition whose group is gone is refused before the marker goes +// on, the way the REST door refuses it: marked and placed, it was a definition +// parented to nothing that the next run then refused over. +func TestSnapshotRestoreRefusesAPreparedTargetWhoseGroupIsGone(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + _ = backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-orphan", ResourceGroupName: "grp-gone", + }) + _ = backend.VolumeDefinitions().Create(ctx, "pvc-orphan", &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 1 << 20}) + }) + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-orphan")); got == 0 { + t.Fatal("restore into a prepared definition whose group is gone succeeded") + } + + if !strings.Contains(errBuf.String(), "no longer exists") { + t.Errorf("refusal %q does not say the group is gone", errBuf.String()) + } + + st := appStore(t, app) + + rd, err := st.ResourceDefinitions().Get(t.Context(), "pvc-orphan") + if err != nil { + t.Fatalf("read the target: %v", err) + } + + if marker := rd.Props[store.RestoreFromSnapshotProp]; marker != "" { + t.Errorf("the refused target was marked %q", marker) + } + + if replicas, _ := st.Resources().ListByDefinition(t.Context(), "pvc-orphan"); len(replicas) != 0 { + t.Errorf("the refused restore placed %d replica(s)", len(replicas)) + } +} + +// The CLI finishing a leftover takes the adoption mark the REST door takes, +// so a rollback of the attempt that created it yields instead of deleting +// what this run finished. +func TestSnapshotRestoreRetryClaimsTheLeftoverItFinishes(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + _ = backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-left", ResourceGroupName: "grp", + Props: map[string]string{store.RestoreFromSnapshotProp: "pvc-x:snap-1"}, + }) + }) + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-left")); got != 0 { + t.Fatalf("retry exit = %d (stderr: %s)", got, errBuf.String()) + } + + rd, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-left") + if err != nil { + t.Fatalf("read the leftover: %v", err) + } + + if rd.Props[store.RestoreAdoptedProp] == "" { + t.Error("the retry finished the leftover without claiming it") + } +} + +// adoptOnFirstReplica plays a concurrent request that adopts the definition +// the moment this command places its first replica, and refuses that +// placement, so this command's rollback runs over a definition someone else +// has already answered for. +type adoptOnFirstReplica struct { + store.ResourceStore + + backend store.Store + fired *atomic.Bool +} + +func (a adoptOnFirstReplica) Create(ctx context.Context, res *apiv1.Resource) error { + if a.fired.CompareAndSwap(false, true) { + if err := store.ClaimAdoptedLeftover(ctx, a.backend, res.Name, &apiv1.Snapshot{}); err != nil { + return err //nolint:wrapcheck // test double + } + + return errPlacementRefused + } + + return a.ResourceStore.Create(ctx, res) //nolint:wrapcheck // pass-through test double +} + +type adoptOnFirstReplicaStore struct { + store.Store + + fired *atomic.Bool +} + +func (a adoptOnFirstReplicaStore) Resources() store.ResourceStore { + return adoptOnFirstReplica{ResourceStore: a.Store.Resources(), backend: a.Store, fired: a.fired} +} + +// The rollback of a definition this command created was a plain delete, so a +// request that adopted the definition meanwhile, and answered for it, lost it. +func TestSnapshotRestoreRollbackLeavesADefinitionAnotherRequestAdopted(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + var ( + errBuf bytes.Buffer + fired atomic.Bool + ) + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { + return adoptOnFirstReplicaStore{Store: backend, fired: &fired}, nil + }, + } + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-adopted")); got == 0 { + t.Fatal("the restore whose placement was refused succeeded") + } + + if !fired.Load() { + t.Fatal("fixture: the concurrent adoption never ran") + } + + // Both sides of the handshake can back off, so the adopter is not + // promised to finish it: the way out is the restore, run again, named as a + // command so a clone that ran it under the hood can follow it too. + if !strings.Contains(errBuf.String(), "another request marked it adopted") || + !strings.Contains(errBuf.String(), "blockstor snapshot resource restore --from-resource pvc-x --from-snapshot snap-1") { + t.Errorf("stderr %q does not say another request took the definition over and how to finish it", errBuf.String()) + } + + rd, err := backend.ResourceDefinitions().Get(t.Context(), "pvc-adopted") + if err != nil { + t.Fatalf("the definition another request adopted was rolled back: %v", err) + } + + if mark, ok := rd.Props[store.RollbackAbandonedProp]; ok { + t.Errorf("the definition left to its adopter still carries rollback mark %q", mark) + } +} + +// failPlacementAndDelete refuses the first replica and the definition delete, +// so the rollback holds the handshake and then cannot delete. +type failPlacementAndDelete struct { + store.Store +} + +type failingFirstReplica struct{ store.ResourceStore } + +func (failingFirstReplica) Create(context.Context, *apiv1.Resource) error { + return errPlacementRefused +} + +type failingRDDelete struct{ store.ResourceDefinitionStore } + +func (failingRDDelete) Delete(context.Context, string) error { + return errPlacementRefused +} + +func (f failPlacementAndDelete) Resources() store.ResourceStore { + return failingFirstReplica{f.Store.Resources()} +} + +func (f failPlacementAndDelete) ResourceDefinitions() store.ResourceDefinitionStore { + return failingRDDelete{f.Store.ResourceDefinitions()} +} + +// A rollback that held the handshake and then could not delete left the +// in-progress mark behind, and nothing was rolling the definition back any +// more: every retry was refused as "being rolled back" instead of finishing it. +func TestSnapshotRestoreRollbackThatCouldNotDeleteReleasesItsMark(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + var errBuf bytes.Buffer + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { + return failPlacementAndDelete{Store: backend}, nil + }, + } + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-undeleted")); got == 0 { + t.Fatal("the restore whose placement was refused succeeded") + } + + rd, err := backend.ResourceDefinitions().Get(t.Context(), "pvc-undeleted") + if err != nil { + t.Fatalf("fixture: the definition the rollback could not delete is gone: %v", err) + } + + if mark, ok := rd.Props[store.RollbackAbandonedProp]; ok { + t.Errorf("a rollback that could not delete left mark %q", mark) + } + + if !strings.Contains(errBuf.String(), "blockstor snapshot resource restore --from-resource pvc-x --from-snapshot snap-1") { + t.Errorf("stderr %q does not say the retry finishes it", errBuf.String()) + } +} + +// Blockstor records what it did to a definition in props it reads back. An +// operator's set-property or delete-property on one of them changes what +// blockstor believes about the definition without it having done anything. +func TestResourceDefinitionPropertyVerbsRefuseServerOwnedKeys(t *testing.T) { + t.Parallel() + + for _, argv := range [][]string{ + {"rd", "set-property", "pvc-owned", store.RestoreFromSnapshotProp, "other:snap"}, + {"rd", "delete-property", "pvc-owned", store.RestoreAdoptedProp}, + {"rd", "set-property", "pvc-owned", store.RestoreVolumesProp, "0:1"}, + {"rd", "set-property", "pvc-owned", store.RollbackAbandonedProp, store.RollbackInProgress}, + } { + t.Run(argv[1], func(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + _ = backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-owned", + Props: map[string]string{ + store.RestoreFromSnapshotProp: "pvc-x:snap-1", + store.RestoreAdoptedProp: "2026-10-07T00:00:00Z", + }, + }) + }) + + if got := app.Run(t.Context(), argv); got == 0 { + t.Fatalf("%v succeeded on a key blockstor sets itself", argv) + } + + if !strings.Contains(errBuf.String(), "blockstor sets this property itself") { + t.Errorf("stderr %q does not say why", errBuf.String()) + } + + rd, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-owned") + if err != nil { + t.Fatalf("read back: %v", err) + } + + if rd.Props[store.RestoreFromSnapshotProp] != "pvc-x:snap-1" || rd.Props[store.RestoreAdoptedProp] == "" { + t.Errorf("the refused edit still changed the props: %v", rd.Props) + } + }) + } +} + +// The CLI restore writes the record of what the snapshot held beside the +// marker, as the REST door does. +func TestSnapshotRestoreRecordsTheSnapshotVolumes(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + }) + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-rec")); got != 0 { + t.Fatalf("restore exit = %d (stderr: %s)", got, errBuf.String()) + } + + rd, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-rec") + if err != nil { + t.Fatalf("read back: %v", err) + } + + if _, ok := store.RecordedRestoreVolumes(rd.Props); !ok { + t.Errorf("the restored definition carries no record: %v", rd.Props) + } +} + +// volumeWrittenConcurrently plays a concurrent run that has already adopted +// the definition this command just created and hydrated a volume into it +// before this command gets to that volume. +type volumeWrittenConcurrently struct { + store.ResourceDefinitionStore + + backend store.Store + sizeKib int64 +} + +func (v volumeWrittenConcurrently) Create(ctx context.Context, rd *apiv1.ResourceDefinition) error { + if err := v.ResourceDefinitionStore.Create(ctx, rd); err != nil { + return err //nolint:wrapcheck // pass-through test double + } + + return v.backend.VolumeDefinitions().Create(ctx, rd.Name, //nolint:wrapcheck // test double + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: v.sizeKib}) +} + +type volumeWrittenConcurrentlyStore struct { + store.Store + + sizeKib int64 +} + +func (v volumeWrittenConcurrentlyStore) ResourceDefinitions() store.ResourceDefinitionStore { + return volumeWrittenConcurrently{ + ResourceDefinitionStore: v.Store.ResourceDefinitions(), backend: v.Store, sizeKib: v.sizeKib, + } +} + +// A run that created the definition met the volume a concurrent run had +// already hydrated into it, took that for a collision and unwound the volumes, +// under a run that had answered for them. +func TestSnapshotRestoreKeepsAVolumeAConcurrentRunHydrated(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + sizeKib int64 + wantOK bool + }{ + {name: "same-size", sizeKib: 1 << 20, wantOK: true}, + {name: "grown", sizeKib: 1 << 21, wantOK: true}, + {name: "smaller", sizeKib: 1 << 19}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + var errBuf bytes.Buffer + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { + return volumeWrittenConcurrentlyStore{Store: backend, sizeKib: tc.sizeKib}, nil + }, + } + + got := app.Run(t.Context(), restoreResourceArgv("pvc-race")) + if (got == 0) != tc.wantOK { + t.Fatalf("exit = %d (stderr: %s), want success=%v", got, errBuf.String(), tc.wantOK) + } + + if !tc.wantOK { + return + } + + vds, err := backend.VolumeDefinitions().List(t.Context(), "pvc-race") + if err != nil || len(vds) != 1 || vds[0].SizeKib != tc.sizeKib { + t.Errorf("volumes after the restore = %v (%v), want the one already there", vds, err) + } + }) + } +} + +// volumeCreateCollides answers AlreadyExists for volume 0. With writeKib set +// it first puts a volume of that size there, the way a concurrent restore +// would; with it zero nothing is there, a volume deleted between the create +// and the read-back. +type volumeCreateCollides struct { + store.VolumeDefinitionStore + + writeKib int64 +} + +func (v volumeCreateCollides) Create(ctx context.Context, rdName string, vd *apiv1.VolumeDefinition) error { + if vd.VolumeNumber != 0 { + return v.VolumeDefinitionStore.Create(ctx, rdName, vd) //nolint:wrapcheck // pass-through test double + } + + if v.writeKib != 0 { + written := apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: v.writeKib} + if err := v.VolumeDefinitionStore.Create(ctx, rdName, &written); err != nil { + return err //nolint:wrapcheck // test double + } + } + + return store.ErrAlreadyExists +} + +type volumeCreateCollidesStore struct { + store.Store + + writeKib int64 +} + +func (v volumeCreateCollidesStore) VolumeDefinitions() store.VolumeDefinitionStore { + return volumeCreateCollides{VolumeDefinitionStore: v.Store.VolumeDefinitions(), writeKib: v.writeKib} +} + +func runOnCollidingVolumes(t *testing.T, backend store.Store, writeKib int64, argv []string) (int, string) { + t.Helper() + + var errBuf bytes.Buffer + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { + return volumeCreateCollidesStore{Store: backend, writeKib: writeKib}, nil + }, + } + + return app.Run(t.Context(), argv), errBuf.String() +} + +// A volume the create says exists and the read-back cannot find was deleted in +// between. Counting it restored would report success over a definition +// without it. +func TestSnapshotRestoreRefusesAVolumeGoneOnReadBack(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + got, stderr := runOnCollidingVolumes(t, backend, 0, restoreResourceArgv("pvc-gone")) + if got == 0 { + t.Fatal("the restore reported success over a volume that is not there") + } + + if !strings.Contains(stderr, "gone when read back") || !strings.Contains(stderr, "run the same command again") { + t.Errorf("stderr %q does not say the volume went away and how to start over", stderr) + } + + if _, err := backend.ResourceDefinitions().Get(t.Context(), "pvc-gone"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the definition this restore created was not rolled back: %v", err) + } +} + +// A resume has no rollback to name the way out, so a volume gone on its +// read-back says which restore finishes the target. +func TestSnapshotRestoreResumeNamesTheRestoreWhenAVolumeIsGoneOnReadBack(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + seedSnapshotSource(ctx, backend) + + snap, err := backend.Snapshots().Get(ctx, "pvc-x", "snap-1") + if err != nil { + t.Fatalf("read the snapshot: %v", err) + } + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-resume", Props: store.WithRestoreMarker(nil, &snap), + }); err != nil { + t.Fatalf("seed the unfinished leftover: %v", err) + } + + got, stderr := runOnCollidingVolumes(t, backend, 0, restoreResourceArgv("pvc-resume")) + if got == 0 { + t.Fatal("the resume reported success over a volume that is not there") + } + + const resume = "blockstor snapshot resource restore --from-resource pvc-x --from-snapshot snap-1 --to-resource pvc-resume" + if !strings.Contains(stderr, "gone when read back") || !strings.Contains(stderr, resume) { + t.Errorf("stderr %q does not name the restore that finishes the target", stderr) + } +} + +// `s vd restore` fails on a volume a concurrent restore of the same snapshot +// wrote between its check and its create, whatever its size, and leaves it as +// it is. Accepting a same-size one let the losing run go on, fail on a later +// volume, and unwind volumes the winning run had answered for: both runs create +// in the snapshot's order, so the loser has to stop on the first volume, with +// nothing of its own to unwind. +func TestSnapshotVolumeDefinitionRestoreFailsOnAVolumeAConcurrentRestoreWrote(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + writeKib int64 + }{ + {name: "same-size", writeKib: 1 << 20}, + {name: "other-size", writeKib: 1 << 19}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + if err := backend.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{Name: "pvc-vd"}); err != nil { + t.Fatalf("seed the target: %v", err) + } + + got, stderr := runOnCollidingVolumes(t, backend, tc.writeKib, []string{ + "s", "vd", "restore", "--from-resource", "pvc-x", "--from-snapshot", "snap-1", "--to-resource", "pvc-vd", + }) + if got == 0 { + t.Fatalf("exit = 0 (stderr: %s), want a failure over the volume another restore wrote", stderr) + } + + vds, err := backend.VolumeDefinitions().List(t.Context(), "pvc-vd") + if err != nil || len(vds) != 1 || vds[0].SizeKib != tc.writeKib { + t.Errorf("volumes = %v (%v), want the one the concurrent restore wrote, untouched", vds, err) + } + }) + } +} + +// replicaGoneOnReadBack answers AlreadyExists to the first replica create and +// has nothing to read back: a replica deleted between the two calls. +type replicaGoneOnReadBack struct { + store.ResourceStore + + fired *atomic.Bool +} + +func (r replicaGoneOnReadBack) Create(ctx context.Context, res *apiv1.Resource) error { + if r.fired.CompareAndSwap(false, true) { + return store.ErrAlreadyExists + } + + return r.ResourceStore.Create(ctx, res) //nolint:wrapcheck // pass-through test double +} + +type replicaGoneOnReadBackStore struct { + store.Store + + fired *atomic.Bool +} + +func (r replicaGoneOnReadBackStore) Resources() store.ResourceStore { + return replicaGoneOnReadBack{ResourceStore: r.Store.Resources(), fired: r.fired} +} + +// A replica the create collided with and that was gone when read back was +// deleted in between, and is created again, as the REST door does. +func TestSnapshotRestoreRecreatesAReplicaGoneOnReadBack(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + var ( + errBuf bytes.Buffer + fired atomic.Bool + ) + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { + return replicaGoneOnReadBackStore{Store: backend, fired: &fired}, nil + }, + } + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-regone", "--nodes", "node-1")); got != 0 { + t.Fatalf("exit = %d (stderr: %s), want the replica created again", got, errBuf.String()) + } + + if _, err := backend.Resources().Get(t.Context(), "pvc-regone", "node-1"); err != nil { + t.Errorf("the replica gone on read-back was not created again: %v", err) + } +} + +// alwaysCollides answers AlreadyExists to every replica create and has +// nothing to read back. +type alwaysCollides struct{ store.ResourceStore } + +func (alwaysCollides) Create(context.Context, *apiv1.Resource) error { return store.ErrAlreadyExists } + +type alwaysCollidesStore struct{ store.Store } + +func (a alwaysCollidesStore) Resources() store.ResourceStore { + return alwaysCollides{a.Store.Resources()} +} + +// A collision that keeps recurring with nothing to read back is not a replica +// placed: counting it one would report the restore done with no replica there. +func TestSnapshotRestoreRefusesAReplicaCollisionThatKeepsRecurring(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + var errBuf bytes.Buffer + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { + return alwaysCollidesStore{backend}, nil + }, + } + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-collide", "--nodes", "node-1")); got == 0 { + t.Fatal("the restore reported success with no replica placed") + } + + if !strings.Contains(errBuf.String(), "kept colliding") { + t.Errorf("stderr %q does not say the collision kept recurring", errBuf.String()) + } + + if _, err := backend.ResourceDefinitions().Get(t.Context(), "pvc-collide"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the definition this restore created was not rolled back: %v", err) + } +} + +// staleReplicaCache collides on the first replica create and serves, from its +// cache, a live replica the API server already holds as being deleted. +type staleReplicaCache struct { + store.ResourceStore + + fired *atomic.Bool +} + +func (s staleReplicaCache) Create(ctx context.Context, res *apiv1.Resource) error { + if s.fired.CompareAndSwap(false, true) { + return store.ErrAlreadyExists + } + + return s.ResourceStore.Create(ctx, res) //nolint:wrapcheck // pass-through test double +} + +func (staleReplicaCache) Get(_ context.Context, rdName, node string) (apiv1.Resource, error) { + return apiv1.Resource{Name: rdName, NodeName: node}, nil +} + +func (staleReplicaCache) GetUncached(_ context.Context, rdName, node string) (apiv1.Resource, error) { + return apiv1.Resource{Name: rdName, NodeName: node, Flags: []string{apiv1.ResourceFlagDelete}}, nil +} + +type staleReplicaCacheStore struct { + store.Store + + fired *atomic.Bool +} + +func (s staleReplicaCacheStore) Resources() store.ResourceStore { + return staleReplicaCache{ResourceStore: s.Store.Resources(), fired: s.fired} +} + +// The replica a create collided with is judged from the API server: a cache +// that trails shows one already accepted for deletion as live. +func TestSnapshotRestoreReadsACollidingReplicaPastTheCache(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + var ( + errBuf bytes.Buffer + fired atomic.Bool + ) + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { + return staleReplicaCacheStore{Store: backend, fired: &fired}, nil + }, + } + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-stale", "--nodes", "node-1")); got == 0 { + t.Fatal("the restore counted a replica being deleted as placed") + } + + if !strings.Contains(errBuf.String(), "still being deleted") { + t.Errorf("stderr %q does not say the replica is being deleted", errBuf.String()) + } +} + +// A diskless replica already on the node holds no copy of the data, and is not +// counted as the restore's own placement. +func TestSnapshotRestoreDoesNotCountADisklessReplicaAsPlaced(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + if err := backend.Resources().Create(t.Context(), &apiv1.Resource{ + Name: "pvc-dl", NodeName: "node-1", Flags: []string{apiv1.ResourceFlagDiskless}, + }); err != nil { + t.Fatalf("seed the diskless replica: %v", err) + } + + var errBuf bytes.Buffer + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { return backend, nil }, + } + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-dl", "--nodes", "node-1")); got == 0 { + t.Fatal("the restore counted a diskless replica as placed") + } + + if !strings.Contains(errBuf.String(), "diskless") { + t.Errorf("stderr %q does not say the replica is diskless", errBuf.String()) + } +} + +// The controller's tie-breaker witness on a node the restore places onto is +// promoted to the replica the restore asked for, with the pool it planned. +func TestSnapshotRestorePromotesATieBreakerOnARequestedNode(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + if err := backend.Resources().Create(t.Context(), &apiv1.Resource{ + Name: "pvc-tb", NodeName: "node-1", + Flags: []string{apiv1.ResourceFlagDiskless, apiv1.ResourceFlagTieBreaker}, + }); err != nil { + t.Fatalf("seed the witness: %v", err) + } + + var errBuf bytes.Buffer + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { return backend, nil }, + } + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-tb", "--nodes", "node-1")); got != 0 { + t.Fatalf("restore onto a node holding the witness = %d (stderr: %s)", got, errBuf.String()) + } + + res, err := backend.Resources().Get(t.Context(), "pvc-tb", "node-1") + if err != nil { + t.Fatalf("read the replica: %v", err) + } + + if !store.HoldsData(&res) || res.Props["StorPoolName"] != "data" { + t.Errorf("witness after the restore: flags %v, pool %q; want a replica with a disk in pool data", + res.Flags, res.Props["StorPoolName"]) + } +} + +// A source stored without a stack is brought up with the default whatever its +// group says now; the restored definition carries that stack, stamped. +func TestSnapshotRestoreStampsTheSourcesDataPlaneStack(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + _ = backend.ResourceGroups().Create(ctx, &apiv1.ResourceGroup{ + Name: "grp-luks", SelectFilter: apiv1.AutoSelectFilter{LayerStack: []string{"DRBD", "LUKS", "STORAGE"}}, + }) + + src, _ := backend.ResourceDefinitions().Get(ctx, "pvc-x") + src.LayerStack = nil + src.ResourceGroupName = "grp-luks" + _ = backend.ResourceDefinitions().Update(ctx, &src) + }) + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-stack")); got != 0 { + t.Fatalf("restore exit = %d (stderr: %s)", got, errBuf.String()) + } + + rd, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-stack") + if err != nil { + t.Fatalf("read the restored definition: %v", err) + } + + if !slices.Equal(rd.LayerStack, apiv1.DefaultLayerStack()) { + t.Errorf("restored stack = %v, want the source's data-plane %v", rd.LayerStack, apiv1.DefaultLayerStack()) + } +} + +// A rollback that left a definition whole and could not take its mark back +// off leaves one every retry refuses. Deleting the mark is the operator's way +// out short of deleting the definition, so that one edit goes through. +func TestResourceDefinitionDeletePropertyClearsAStuckRollbackMark(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + _ = backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-stuck", + Props: map[string]string{ + store.RestoreFromSnapshotProp: "pvc-x:snap-1", + store.RollbackAbandonedProp: store.RollbackInProgress, + }, + }) + }) + + if got := app.Run(t.Context(), []string{"rd", "delete-property", "pvc-stuck", store.RollbackAbandonedProp}); got != 0 { + t.Fatalf("delete-property of the rollback mark = exit %d (stderr: %s)", got, errBuf.String()) + } + + rd, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-stuck") + if err != nil { + t.Fatalf("read back: %v", err) + } + + if mark, ok := rd.Props[store.RollbackAbandonedProp]; ok { + t.Errorf("the mark is still there: %q", mark) + } + + if rd.Props[store.RestoreFromSnapshotProp] != "pvc-x:snap-1" { + t.Errorf("clearing the mark changed the marker: %v", rd.Props) + } +} + +// Only a mark over a definition the rollback left whole is the operator's to +// clear: one naming a later step a rollback gave up at stays, on both verbs +// that delete. Setting the key to +// nothing is the CLI's delete, and clears an in-progress mark the same way. +func TestResourceDefinitionPropertyVerbsClearOnlyAMarkOverAWholeDefinition(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + mark string + argv []string + clear bool + other bool + }{ + {name: "delete-step", mark: "reap-replicas", argv: []string{"rd", "delete-property", "pvc-m", store.RollbackAbandonedProp}}, + {name: "set-empty-step", mark: "reap-replicas", argv: []string{"rd", "set-property", "pvc-m", store.RollbackAbandonedProp}}, + { + name: "set-empty-in-progress", mark: store.RollbackInProgress, clear: true, + argv: []string{"rd", "set-property", "pvc-m", store.RollbackAbandonedProp}, + }, + { + name: "delete-snapshots-step", mark: store.RollbackStepSnapshots, clear: true, + argv: []string{"rd", "delete-property", "pvc-m", store.RollbackAbandonedProp}, + }, + {name: "other-key-on-a-step-mark", mark: "reap-replicas", other: true, argv: []string{"rd", "set-property", "pvc-m", "Aux/x", "1"}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + _ = backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-m", Props: map[string]string{store.RollbackAbandonedProp: tc.mark}, + }) + }) + + got := app.Run(t.Context(), tc.argv) + + rd, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-m") + if err != nil { + t.Fatalf("read back: %v", err) + } + + _, kept := rd.Props[store.RollbackAbandonedProp] + + if tc.other { + if got != 0 || !kept || rd.Props["Aux/x"] != "1" { + t.Errorf("%v = exit %d, props %v (stderr: %s), want the edit through and the mark kept", + tc.argv, got, rd.Props, errBuf.String()) + } + + return + } + + if tc.clear { + if got != 0 || kept { + t.Errorf("%v = exit %d, mark kept %v (stderr: %s), want it cleared", tc.argv, got, kept, errBuf.String()) + } + + return + } + + if got == 0 || !kept { + t.Errorf("%v = exit %d, mark kept %v, want it refused and kept", tc.argv, got, kept) + } + }) + } +} + +// markerLandsLate hides the restore marker from every read until this run +// writes its own stamp: a concurrent run of the same restore stamped it in +// between, so this run judged the target prepared and then found the marker +// already there. +type markerLandsLate struct { + store.ResourceDefinitionStore + + stamped *atomic.Bool +} + +func (m markerLandsLate) Get(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + return m.hide(m.ResourceDefinitionStore.Get(ctx, name)) +} + +func (m markerLandsLate) GetUncached(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + return m.hide(m.ResourceDefinitionStore.GetUncached(ctx, name)) +} + +func (m markerLandsLate) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + m.stamped.Store(true) + + return m.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) //nolint:wrapcheck // test double +} + +func (m markerLandsLate) hide(rd apiv1.ResourceDefinition, err error) (apiv1.ResourceDefinition, error) { + if err == nil && !m.stamped.Load() { + rd.Props = maps.Clone(rd.Props) + delete(rd.Props, store.RestoreFromSnapshotProp) + delete(rd.Props, store.RestoreVolumesProp) + } + + return rd, err +} + +type markerLandsLateStore struct { + store.Store + + stamped *atomic.Bool +} + +func (s markerLandsLateStore) ResourceDefinitions() store.ResourceDefinitionStore { + return markerLandsLate{ResourceDefinitionStore: s.Store.ResourceDefinitions(), stamped: s.stamped} +} + +func (s markerLandsLateStore) Resources() store.ResourceStore { + return replicaLandsLate{ResourceStore: s.Store.Resources(), stamped: s.stamped} +} + +// replicaLandsLate hides the concurrent run's replica along with its marker: +// that run placed it after this one judged the target prepared. +type replicaLandsLate struct { + store.ResourceStore + + stamped *atomic.Bool +} + +func (r replicaLandsLate) ListByDefinition(ctx context.Context, rdName string) ([]apiv1.Resource, error) { + replicas, err := r.ResourceStore.ListByDefinition(ctx, rdName) + if err != nil || r.stamped.Load() { + return replicas, err //nolint:wrapcheck // test double + } + + return slices.DeleteFunc(replicas, func(res apiv1.Resource) bool { return res.NodeName == "node-1" }), nil +} + +// A run that finds the marker already on a target it judged prepared lost the +// stamp to a concurrent run of the same restore. It judges that run's +// definition as a leftover, and a finished one is left as it is rather than +// placed over with this run's nodes. +func TestSnapshotRestoreJudgesATargetAConcurrentRunMarkedFirst(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + seedSnapshotSource(ctx, backend) + + snap, err := backend.Snapshots().Get(ctx, "pvc-x", "snap-1") + if err != nil { + t.Fatalf("read the snapshot: %v", err) + } + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-late", ResourceGroupName: "grp", LayerStack: []string{"DRBD", "STORAGE"}, + Props: store.WithRestoreMarker(nil, &snap), + }); err != nil { + t.Fatalf("seed the target the other run marked: %v", err) + } + + if err := backend.VolumeDefinitions().Create(ctx, "pvc-late", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 1 << 20}); err != nil { + t.Fatalf("seed the target's volume: %v", err) + } + + if err := backend.Resources().Create(ctx, &apiv1.Resource{ + Name: "pvc-late", NodeName: "node-1", Props: map[string]string{"StorPoolName": "data"}, + }); err != nil { + t.Fatalf("seed the other run's replica: %v", err) + } + + var ( + errBuf bytes.Buffer + stamped atomic.Bool + ) + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { + return markerLandsLateStore{Store: backend, stamped: &stamped}, nil + }, + } + + if got := app.Run(ctx, restoreResourceArgv("pvc-late", "--nodes", "node-2")); got != 0 { + t.Fatalf("restore over a finished target another run marked = %d (stderr: %s)", got, errBuf.String()) + } + + if _, err := backend.Resources().Get(ctx, "pvc-late", "node-2"); err == nil { + t.Errorf("the run placed over the other run's finished restore with its own nodes") + } +} + +var errWitnessPatchRefused = errors.New("apiserver refused the witness patch") + +// witnessPatchFaults fails the patch that would promote a witness, or shows +// the witness already being deleted when the patch reads it. +type witnessPatchFaults struct { + store.ResourceStore + + refuse, deleting, replaced bool +} + +func (w witnessPatchFaults) PatchResourceSpec( + ctx context.Context, rdName, node string, mutate func(*apiv1.Resource) error, +) error { + if w.refuse { + return errWitnessPatchRefused + } + + return w.ResourceStore.PatchResourceSpec(ctx, rdName, node, func(live *apiv1.Resource) error { //nolint:wrapcheck // test double + if w.deleting { + live.Flags = append(live.Flags, apiv1.ResourceFlagDelete) + } + + if w.replaced { + live.Flags = []string{apiv1.ResourceFlagDiskless} + } + + return mutate(live) + }) +} + +type witnessPatchFaultStore struct { + store.Store + + refuse, deleting, replaced bool +} + +func (s witnessPatchFaultStore) Resources() store.ResourceStore { + return witnessPatchFaults{ + ResourceStore: s.Store.Resources(), refuse: s.refuse, deleting: s.deleting, replaced: s.replaced, + } +} + +// A witness the restore could not promote, because the patch failed, the +// witness started going away under it, or an operator's diskless replica took +// its place, fails the restore: reported done, it +// would leave a diskless witness where the restore asked for its data. +func TestSnapshotRestoreFailsOnAWitnessItCouldNotPromote(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + refuse, deleting, replaced bool + }{ + {name: "patch-refused", refuse: true}, + {name: "being-deleted", deleting: true}, + {name: "replaced-by-a-diskless", replaced: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedSnapshotSource(t.Context(), backend) + + if err := backend.Resources().Create(t.Context(), &apiv1.Resource{ + Name: "pvc-tbf", NodeName: "node-1", + Flags: []string{apiv1.ResourceFlagDiskless, apiv1.ResourceFlagTieBreaker}, + }); err != nil { + t.Fatalf("seed the witness: %v", err) + } + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &bytes.Buffer{}, + StoreFor: func(context.Context) (store.Store, error) { + return witnessPatchFaultStore{ + Store: backend, refuse: tc.refuse, deleting: tc.deleting, replaced: tc.replaced, + }, nil + }, + } + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-tbf", "--nodes", "node-1")); got == 0 { + t.Error("the restore reported success over a witness it could not promote") + } + }) + } +} + +// A re-run that finishes a leftover records the volumes of the snapshot it +// restores from, replacing a record an earlier attempt left from another one. +func TestSnapshotRestoreResumeRecordsTheVolumesOfItsSnapshot(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + seedSnapshotSource(ctx, backend) + + snap, err := backend.Snapshots().Get(ctx, "pvc-x", "snap-1") + if err != nil { + t.Fatalf("read the snapshot: %v", err) + } + + props := store.WithRestoreMarker(nil, &snap) + props[store.RestoreVolumesProp] = store.EncodeRestoreVolumes([]apiv1.SnapshotVolumeDef{ + {VolumeNumber: 0, SizeKib: 1 << 20}, {VolumeNumber: 1, SizeKib: 1 << 10}, + }) + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-rec", ResourceGroupName: "grp", LayerStack: []string{"DRBD", "STORAGE"}, Props: props, + }); err != nil { + t.Fatalf("seed the leftover with a stale record: %v", err) + } + + var errBuf bytes.Buffer + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { return backend, nil }, + } + + if got := app.Run(ctx, restoreResourceArgv("pvc-rec", "--nodes", "node-1")); got != 0 { + t.Fatalf("resume of the leftover = %d (stderr: %s)", got, errBuf.String()) + } + + rd, err := backend.ResourceDefinitions().Get(ctx, "pvc-rec") + if err != nil { + t.Fatalf("read the restored definition: %v", err) + } + + if want := store.EncodeRestoreVolumes(snap.VolumeDefinitions); rd.Props[store.RestoreVolumesProp] != want { + t.Errorf("volume record = %q, want %q", rd.Props[store.RestoreVolumesProp], want) + } +} + +// `s vd restore` into a definition another restore has marked as its own is +// refused before anything is written: that restore hydrates the volumes +// itself, and one this command wrote and then unwound would go from under it. +func TestSnapshotVolumeDefinitionRestoreRefusesATargetUnderARestoreMarker(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + seedSnapshotSource(ctx, backend) + + snap, err := backend.Snapshots().Get(ctx, "pvc-x", "snap-1") + if err != nil { + t.Fatalf("read the snapshot: %v", err) + } + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-marked", Props: store.WithRestoreMarker(nil, &snap), + }); err != nil { + t.Fatalf("seed the marked target: %v", err) + } + + var errBuf bytes.Buffer + + app := &cli.App{ + Out: &bytes.Buffer{}, + Err: &errBuf, + StoreFor: func(context.Context) (store.Store, error) { return backend, nil }, + } + + if got := app.Run(ctx, []string{ + "s", "vd", "restore", "--from-resource", "pvc-x", "--from-snapshot", "snap-1", "--to-resource", "pvc-marked", + }); got == 0 { + t.Fatal("volume-definition restore into a marked target succeeded") + } + + if vds, _ := backend.VolumeDefinitions().List(ctx, "pvc-marked"); len(vds) != 0 { + t.Errorf("the refused restore wrote %d volume(s)", len(vds)) + } +} diff --git a/internal/cli/snapshot_restore_without_snapshot_test.go b/internal/cli/snapshot_restore_without_snapshot_test.go new file mode 100644 index 00000000..64117231 --- /dev/null +++ b/internal/cli/snapshot_restore_without_snapshot_test.go @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli_test + +import ( + "context" + "strings" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// A re-run over a finished restore needs neither the snapshot nor the source: +// both can be cleaned up once the restore is done. +func TestSnapshotRestoreRerunSurvivesTheSnapshotAndTheSource(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + remove func(ctx context.Context, backend store.Store) error + }{ + {name: "snapshot-deleted", remove: func(ctx context.Context, backend store.Store) error { + return backend.Snapshots().Delete(ctx, "pvc-x", "snap-1") + }}, + {name: "source-deleted", remove: func(ctx context.Context, backend store.Store) error { + return backend.ResourceDefinitions().Delete(ctx, "pvc-x") + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + app, outBuf, errBuf := newApp(t, seedSnapshotSource) + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-done", "--nodes", "node-1")); got != 0 { + t.Fatalf("first restore exit = %d (stderr: %s)", got, errBuf.String()) + } + + backend := appStore(t, app) + if err := tc.remove(t.Context(), backend); err != nil { + t.Fatalf("fixture: %v", err) + } + + before, _ := backend.Resources().ListByDefinition(t.Context(), "pvc-done") + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-done", "--nodes", "node-1")); got != 0 { + t.Fatalf("re-run exit = %d (stderr: %s), want the finished restore left alone", got, errBuf.String()) + } + + if !strings.Contains(outBuf.String(), "pvc-done is already restored") { + t.Errorf("stdout %q does not say the restore is already done", outBuf.String()) + } + + after, _ := backend.Resources().ListByDefinition(t.Context(), "pvc-done") + if len(after) != len(before) { + t.Errorf("the re-run placed replicas: %d before, %d after", len(before), len(after)) + } + }) + } +} + +// An unfinished restore cannot be finished from a snapshot that is gone, and +// the refusal names the way out. +func TestSnapshotRestoreRefusesAnUnfinishedTargetWhoseSnapshotIsGone(t *testing.T) { + t.Parallel() + + snap := apiv1.Snapshot{ + Name: "snap-1", ResourceName: "pvc-x", + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 1 << 20}}, + } + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + _ = backend.Snapshots().Delete(ctx, "pvc-x", "snap-1") + _ = backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-half", Props: store.WithRestoreMarker(nil, &snap), + }) + }) + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-half", "--nodes", "node-1")); got == 0 { + t.Fatal("the restore over an unfinished target with its snapshot gone succeeded") + } + + if !strings.Contains(errBuf.String(), "restore it from another snapshot") { + t.Errorf("stderr %q does not name the way out", errBuf.String()) + } + + if vds, _ := appStore(t, app).VolumeDefinitions().List(t.Context(), "pvc-half"); len(vds) != 0 { + t.Errorf("the refused restore hydrated %d volume(s)", len(vds)) + } +} + +// With no target to judge, a restore needs its source and its snapshot, and a +// missing one is the error, before anything is written. +func TestSnapshotRestoreNeedsItsSourceWhenThereIsNoTarget(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + remove func(ctx context.Context, backend store.Store) + stderr string + }{ + {name: "source-gone", stderr: "get resource definition pvc-x", remove: func(ctx context.Context, backend store.Store) { + _ = backend.ResourceDefinitions().Delete(ctx, "pvc-x") + }}, + {name: "snapshot-gone", stderr: "get snapshot snap-1 of pvc-x", remove: func(ctx context.Context, backend store.Store) { + _ = backend.Snapshots().Delete(ctx, "pvc-x", "snap-1") + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + tc.remove(ctx, backend) + }) + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-new", "--nodes", "node-1")); got == 0 { + t.Fatal("the restore succeeded without its source") + } + + if !strings.Contains(errBuf.String(), tc.stderr) { + t.Errorf("stderr %q does not name %q", errBuf.String(), tc.stderr) + } + + if _, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-new"); err == nil { + t.Error("the failed restore created its target") + } + }) + } +} + +// A target restored before the volumes were recorded has nothing to be judged +// against once its snapshot is gone, and keeps the error it got. +func TestSnapshotRestoreKeepsALegacyTargetsErrorWhenTheSnapshotIsGone(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + _ = backend.Snapshots().Delete(ctx, "pvc-x", "snap-1") + _ = backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-legacy", + Props: map[string]string{store.RestoreFromSnapshotProp: "pvc-x:snap-1"}, + }) + _ = backend.VolumeDefinitions().Create(ctx, "pvc-legacy", &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 1 << 20}) + _ = backend.Resources().Create(ctx, &apiv1.Resource{Name: "pvc-legacy", NodeName: "node-1"}) + }) + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-legacy", "--nodes", "node-1")); got == 0 { + t.Fatal("a legacy target with its snapshot gone was reported restored") + } + + if !strings.Contains(errBuf.String(), "get snapshot snap-1 of pvc-x") { + t.Errorf("stderr %q does not name the missing snapshot", errBuf.String()) + } +} diff --git a/internal/cli/write.go b/internal/cli/write.go index 4f2ac1d3..35ec6a18 100644 --- a/internal/cli/write.go +++ b/internal/cli/write.go @@ -59,12 +59,12 @@ type propertyAccessor struct { // touched. The edit guards judge the resulting state, which cannot // see an operator setting an immutable key to the value it already // has — REST refuses that, so this door has to as well. - namedKey func(key string) error + namedKey func(key string, deleting bool) error } // withNamedKeyGuard attaches a named-key guard to an accessor built by // objectProps, whose signature carries only the state-comparing guards. -func withNamedKeyGuard(accessor propertyAccessor, guard func(string) error) propertyAccessor { +func withNamedKeyGuard(accessor propertyAccessor, guard func(string, bool) error) propertyAccessor { accessor.namedKey = guard return accessor @@ -87,18 +87,18 @@ func setProperty(accessor propertyAccessor) handler { ident := run.Flags.Positionals[:accessor.args] key := run.Flags.Positionals[accessor.args] + value := "" + if len(run.Flags.Positionals) > want { + value = run.Flags.Positionals[want] + } + if accessor.namedKey != nil { - named := accessor.namedKey(key) + named := accessor.namedKey(key, value == "") if named != nil { return named } } - value := "" - if len(run.Flags.Positionals) > want { - value = run.Flags.Positionals[want] - } - return accessor.edit(ctx, run.Store, ident, func(props map[string]string) error { if value == "" { delete(props, key) @@ -211,10 +211,11 @@ func objectProps[T any]( } } -// rdProps accesses a resource definition's property bag. +// rdProps accesses a resource definition's property bag. The props blockstor +// writes to record what a definition is are refused here as on the REST door. // -//nolint:gochecknoglobals,dupl // static accessor table; the parallel shape is the point -var rdProps = objectProps("resource definition", 1, +//nolint:gochecknoglobals // static accessor table +var rdProps = withNamedKeyGuard(objectProps("resource definition", 1, func(ctx context.Context, st store.Store, ident []string) (apiv1.ResourceDefinition, error) { return st.ResourceDefinitions().Get(ctx, ident[0]) }, @@ -222,7 +223,31 @@ var rdProps = objectProps("resource definition", 1, func(ctx context.Context, st store.Store, ident []string, mutate func(*apiv1.ResourceDefinition) error) error { return st.ResourceDefinitions().PatchResourceDefinitionSpec(ctx, ident[0], mutate) }, -) + store.RollbackMarkClearRefusal, +), refuseServerOwnedDefinitionProp) + +// errServerOwnedProp refuses an operator's edit of a prop blockstor sets on a +// definition itself. +var errServerOwnedProp = errors.New("blockstor sets this property itself") + +// refuseServerOwnedDefinitionProp is the named-key guard for rdProps. A +// delete of the rollback mark goes through; see +// store.ServerOwnedPropEditByOperator. +func refuseServerOwnedDefinitionProp(key string, deleting bool) error { + // The value is not judged, only that one is set; an empty override is + // read as a delete, so a set is spelled with a value. + var overrides map[string]string + if !deleting { + overrides = map[string]string{key: "set"} + } + + if store.ServerOwnedPropEditByOperator(overrides, []string{key}, nil) != "" { + return fmt.Errorf("%w: %s records what blockstor did to this definition and is read back "+ + "to decide what to do next", errServerOwnedProp, key) + } + + return nil +} // nodeProps accesses a node's property bag. // diff --git a/pkg/rest/adopted_leftover_rollback_test.go b/pkg/rest/adopted_leftover_rollback_test.go new file mode 100644 index 00000000..ee414c1b --- /dev/null +++ b/pkg/rest/adopted_leftover_rollback_test.go @@ -0,0 +1,613 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "context" + "encoding/json" + "maps" + "net/http" + "strings" + "sync/atomic" + "testing" + + "github.com/cockroachdb/errors" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +var errHeldStampFailed = errors.New("held replica create failed") + +// firstStampHeld holds the first replica Create until released, then fails +// it: the first attempt at an operation stalls on its stamp while a retry runs +// to completion, and only then fails. +type firstStampHeld struct { + store.Store + + calls *atomic.Int32 + entered chan struct{} + release chan struct{} +} + +type firstStampHeldResources struct { + store.ResourceStore + + outer firstStampHeld +} + +func (f firstStampHeld) Resources() store.ResourceStore { + return firstStampHeldResources{ResourceStore: f.Store.Resources(), outer: f} +} + +func (r firstStampHeldResources) Create(ctx context.Context, res *apiv1.Resource) error { + if r.outer.calls.Add(1) == 1 { + close(r.outer.entered) + <-r.outer.release + + return errHeldStampFailed + } + + return r.ResourceStore.Create(ctx, res) //nolint:wrapcheck // pass-through test double +} + +func newFirstStampHeld(backend store.Store) firstStampHeld { + return firstStampHeld{ + Store: backend, calls: &atomic.Int32{}, + entered: make(chan struct{}), release: make(chan struct{}), + } +} + +// A retry adopts the definition an earlier attempt created while that attempt +// is still running, and answers 201. The earlier attempt then failed, and its +// rollback cascaded a delete over everything under the name: the caller held +// a 201 for a volume that no longer existed. +func TestTheCreatorsRollbackLeavesWhatARetryAdoptedAndAnsweredFor(t *testing.T) { + t.Parallel() + + for _, door := range []struct { + name string + target string + post func(t *testing.T, base string) int + }{ + {name: "clone", target: "dst-adopt-c", post: func(t *testing.T, base string) int { + t.Helper() + + return cloneOnce(t, base, "src-adopt-c", "dst-adopt-c", nil) + }}, + {name: "restore", target: "dst-adopt-r", post: func(t *testing.T, base string) int { + t.Helper() + + raw, err := json.Marshal(map[string]any{"to_resource": "dst-adopt-r", "node_names": []string{"node-a"}}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + resp := httpPost(t, base+"/v1/resource-definitions/src-adopt-r/snapshot-restore-resource/snap-adopt-r", raw) + _ = resp.Body.Close() + + return resp.StatusCode + }}, + } { + t.Run(door.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + src := "src-adopt-" + door.name[:1] + seedDeployedCloneSource(t, backend, src) + seedRestoreSnapshot(t, backend, src, "snap-adopt-"+door.name[:1], []string{"node-a"}) + + held := newFirstStampHeld(backend) + + base, stop := startServerWithStore(t, held) + defer stop() + + first := make(chan int, 1) + + go func() { first <- door.post(t, base) }() + + <-held.entered + + if code := door.post(t, base); code != http.StatusCreated { + t.Fatalf("the retry that adopted the leftover = %d, want 201", code) + } + + close(held.release) + + if code := <-first; code == http.StatusCreated { + t.Fatalf("the first attempt, whose stamp failed, = 201") + } + + if _, err := backend.ResourceDefinitions().Get(ctx, door.target); err != nil { + t.Fatalf("the definition the retry answered 201 for is gone: %v", err) + } + + if replicas, _ := backend.Resources().ListByDefinition(ctx, door.target); len(replicas) == 0 { + t.Errorf("the replica the retry answered 201 for is gone") + } + }) + } +} + +// An adoption that finds the creator's rollback already started refuses +// rather than writing into a definition that is being taken apart. +func TestAnAdoptionRefusesALeftoverWhoseCreatorIsRollingBack(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-rolling", Props: map[string]string{rollbackAbandonedKey: rollbackInProgress}, + }); err != nil { + t.Fatalf("seed the definition: %v", err) + } + + s := &Server{Store: st} + + if err := s.claimAdoptedLeftover(ctx, "dst-rolling", &apiv1.Snapshot{}); !errors.Is(err, errAdoptedLeftoverRollingBack) { + t.Errorf("adoption over a rollback in progress = %v, want a refusal", err) + } +} + +// secondStampHeld holds the stamp of one node's replica of one definition until +// released, then fails it: a clone has placed its first replica, and so reads +// as finished, while the attempt that created it is still stamping the next. +type secondStampHeld struct { + store.Store + + target, node string + entered, release chan struct{} +} + +type secondStampHeldResources struct { + store.ResourceStore + + outer secondStampHeld +} + +func (h secondStampHeld) Resources() store.ResourceStore { + return secondStampHeldResources{ResourceStore: h.Store.Resources(), outer: h} +} + +func (r secondStampHeldResources) Create(ctx context.Context, res *apiv1.Resource) error { + if res.Name == r.outer.target && res.NodeName == r.outer.node { + select { + case r.outer.entered <- struct{}{}: + <-r.outer.release + + return errHeldStampFailed + default: + } + } + + return r.ResourceStore.Create(ctx, res) //nolint:wrapcheck // pass-through test double +} + +// A replay and the status poll answer a clone finished the moment it has its +// volumes and a replica, and neither adopts it, since the answer writes +// nothing. That moment comes while the attempt that created it is still +// stamping its other replicas, so when the next stamp fails the rollback has +// to see the clone the way they did: deleting it took a volume the driver had +// just been told exists. +func TestTheCreatorsRollbackLeavesACloneAReplayAnsweredFinished(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedTwoNodeCloneSource(t, backend, "src-answered") + + held := secondStampHeld{ + Store: backend, target: "dst-answered", node: "node-b", + entered: make(chan struct{}), release: make(chan struct{}), + } + + base, stop := startServerWithStore(t, held) + defer stop() + + first := make(chan int, 1) + + go func() { first <- cloneOnce(t, base, "src-answered", "dst-answered", nil) }() + + <-held.entered + + if code := cloneOnce(t, base, "src-answered", "dst-answered", nil); code != http.StatusCreated { + t.Fatalf("replay while the first attempt stamps node-b = %d, want 201", code) + } + + if _, got := cloneStatusOnce(t, base+"/v1/resource-definitions/src-answered/clone/dst-answered"); got != "COMPLETE" { + t.Fatalf("status poll while the first attempt stamps node-b = %q, want COMPLETE", got) + } + + close(held.release) + + if code := <-first; code != http.StatusInternalServerError { + t.Errorf("first attempt whose node-b stamp failed = %d, want 500", code) + } + + rd, err := backend.ResourceDefinitions().Get(t.Context(), "dst-answered") + if err != nil { + t.Fatalf("the clone a replay answered 201 for was rolled back: %v", err) + } + + if mark := rd.Props[store.RollbackAbandonedProp]; mark != "" { + t.Errorf("the clone left in place carries rollback mark %q, which refuses every retry", mark) + } + + if code := cloneOnce(t, base, "src-answered", "dst-answered", nil); code != http.StatusCreated { + t.Errorf("replay after the first attempt failed = %d, want 201", code) + } +} + +// A placed restore whose node-b replica fails has placed node-a, so it already +// reads as finished and the rollback leaves it. A retry judges it finished too +// and places nothing, so the refusal names the replica it never placed and the +// command that places it. +func TestTheRestoreLeftShortNamesTheReplicaItNeverPlaced(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedTwoNodeSource(t, backend, "src-short") + seedRestoreSnapshot(t, backend, "src-short", "snap-short", []string{"node-a", "node-b"}) + + base, stop := startServerWithStore(t, halfPlacingStore{Store: backend, target: "dst-short"}) + defer stop() + + code, rc := restoreAnswer(t, base, "src-short", "snap-short", map[string]any{ + "to_resource": "dst-short", "nodes": []string{"node-a", "node-b"}, + }) + if code != http.StatusInternalServerError { + t.Fatalf("restore whose node-b replica failed = %d %q, want 500", code, rc.Message) + } + + if !strings.Contains(rc.Correc, "no replica on node-b") || !strings.Contains(rc.Correc, "linstor resource create") { + t.Errorf("correction %q does not name the replica the restore never placed", rc.Correc) + } + + rd, err := backend.ResourceDefinitions().Get(t.Context(), "dst-short") + if err != nil { + t.Fatalf("the restore a retry may have answered for was rolled back: %v", err) + } + + if mark := rd.Props[store.RollbackAbandonedProp]; mark != "" { + t.Errorf("the definition left in place carries rollback mark %q", mark) + } +} + +// The scope decides what a failed materialisation's rollback spares, and both +// sides of it are pinned: a definition that reads as finished on the door's +// own terms is left, one that does not is deleted. +func TestTheRollbackScopeSparesOnlyWhatReadsAsFinished(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + scope rollbackScope + replica bool + wantSpare bool + }{ + {name: "bare-restore-hydrated", scope: rollbackUnlessHydrated, wantSpare: true}, + {name: "placed-without-a-replica", scope: rollbackUnlessPlaced}, + {name: "placed-with-a-replica", scope: rollbackUnlessPlaced, replica: true, wantSpare: true}, + {name: "rg-deleted", scope: rollbackEvenIfFinished, replica: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "dst-scope"}); err != nil { + t.Fatalf("seed the definition: %v", err) + } + + if err := backend.VolumeDefinitions().Create(ctx, "dst-scope", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 1024}); err != nil { + t.Fatalf("seed its volume: %v", err) + } + + var placed []string + + if tc.replica { + if err := backend.Resources().Create(ctx, &apiv1.Resource{Name: "dst-scope", NodeName: "node-a"}); err != nil { + t.Fatalf("seed its replica: %v", err) + } + + placed = []string{"node-a"} + } + + err := (&Server{Store: backend}).rollBackCompensating(ctx, "dst-scope", placed, tc.scope) + + _, getErr := backend.ResourceDefinitions().Get(ctx, "dst-scope") + + if tc.wantSpare { + if !errors.Is(err, errRollbackAnswered) || getErr != nil { + t.Errorf("rollback = %v, definition read = %v: want it left as finished", err, getErr) + } + + return + } + + if err != nil || !errors.Is(getErr, store.ErrNotFound) { + t.Errorf("rollback = %v, definition read = %v: want it deleted", err, getErr) + } + }) + } +} + +// linstor-csi deletes the source's Aux/csi-provisioning-completed-by on every +// clone, and its CreateVolume takes a definition carrying it, with volumes of +// the right size, as one already provisioned: the retry never reaches the +// clone that would resume it. A leftover a failed clone keeps must not carry +// it, so the edits land with the definition, not after the clone finished. +func TestACloneLeftoverDoesNotCarryAPropTheCallerDeleted(t *testing.T) { + t.Parallel() + + const completedBy = "Aux/csi-provisioning-completed-by" + + for _, tc := range []struct { + name string + fails bool + edits map[string]any + }{ + {name: "left-by-a-failed-clone", fails: true}, + {name: "finished-clone"}, + {name: "namespace-deleted", fails: true, edits: map[string]any{"delete_namespaces": []string{"Aux"}}}, + {name: "override-kept", fails: true, edits: map[string]any{ + "delete_props": []string{completedBy}, + "override_props": map[string]string{"Aux/owner": "csi"}, + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedTwoNodeSource(t, backend, "src-csi") + + if err := backend.ResourceDefinitions().PatchResourceDefinitionSpec(ctx, "src-csi", + func(rd *apiv1.ResourceDefinition) error { + if rd.Props == nil { + rd.Props = map[string]string{} + } + + rd.Props[completedBy] = "linstor-csi/v1.10.4" + + return nil + }); err != nil { + t.Fatalf("mark the source provisioned: %v", err) + } + + var st store.Store = backend + if tc.fails { + st = halfPlacingStore{Store: backend, target: "dst-csi"} + } + + base, stop := startServerWithStore(t, st) + defer stop() + + body := map[string]any{"name": "dst-csi", "use_zfs_clone": true, "delete_props": []string{completedBy}} + if tc.edits != nil { + body = map[string]any{"name": "dst-csi", "use_zfs_clone": true} + maps.Copy(body, tc.edits) + } + + resp := postClone(t, base, "src-csi", body) + _ = resp.Body.Close() + + rd, err := backend.ResourceDefinitions().Get(ctx, "dst-csi") + if err != nil { + t.Fatalf("clone = %d, and no target to inspect: %v", resp.StatusCode, err) + } + + if v, ok := rd.Props[completedBy]; ok { + t.Errorf("clone = %d left a target carrying %s=%q, which linstor-csi reads as provisioned", + resp.StatusCode, completedBy, v) + } + + if tc.name == "override-kept" && rd.Props["Aux/owner"] != "csi" { + t.Errorf("clone = %d left a target without the override the caller asked for: %v", + resp.StatusCode, rd.Props) + } + }) + } +} + +// A bare restore's replay judges a leftover with every volume and no replica +// finished and answers 201, while a placed restore of the same snapshot into +// the same name, still placing, judges it unfinished and rolls it back. The +// replay takes the adoption mark before it answers, so that rollback yields +// instead of deleting the definition the replay was answered for. +func TestABareReplayKeepsAPlacedRestoreFromRollingItBack(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-bare") + seedRestoreSnapshot(t, st, "src-bare", "snap-bare", []string{"node-a"}) + + snap, err := st.Snapshots().Get(ctx, "src-bare", "snap-bare") + if err != nil { + t.Fatalf("read the snapshot: %v", err) + } + + seedRestoreLeftover(t, st, "src-bare", "snap-bare", "dst-bare", store.WithRestoreMarker(nil, &snap)) + + if err := st.VolumeDefinitions().Create(ctx, "dst-bare", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the hydrated volume: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := restoreOnce(t, base, "src-bare", "snap-bare", map[string]any{"to_resource": "dst-bare"}); code != http.StatusCreated { + t.Fatalf("bare replay of the hydrated leftover = %d, want 201", code) + } + + err = (&Server{Store: st}).rollBackCompensating(ctx, "dst-bare", []string{"node-a"}, rollbackUnlessPlaced) + if !errors.Is(err, errRollbackYielded) { + t.Errorf("placed restore's rollback after the bare replay = %v, want it to yield", err) + } + + if _, err := st.ResourceDefinitions().Get(ctx, "dst-bare"); err != nil { + t.Errorf("the definition the replay answered 201 for is gone: %v", err) + } +} + +// rollbackStartsUnderTheClaim lands the creator's in-progress rollback mark on +// the target just before the replay's adoption patch: the creator began rolling +// back after the replay's gate and before its claim. +type rollbackStartsUnderTheClaim struct { + store.ResourceDefinitionStore + + target string +} + +func (r rollbackStartsUnderTheClaim) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + if name == r.target { + if err := r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, + func(rd *apiv1.ResourceDefinition) error { + rd.Props[store.RollbackAbandonedProp] = store.RollbackInProgress + + return nil + }); err != nil { + return err //nolint:wrapcheck // test double + } + } + + return r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) //nolint:wrapcheck // test double +} + +type rollbackStartsUnderTheClaimStore struct { + store.Store + + target string +} + +func (s rollbackStartsUnderTheClaimStore) ResourceDefinitions() store.ResourceDefinitionStore { + return rollbackStartsUnderTheClaim{ResourceDefinitionStore: s.Store.ResourceDefinitions(), target: s.target} +} + +// A replay whose claim finds the creator rolling back refuses with 409 in the +// FAIL_EXISTS_RSC_DFN band, the answer the gate gives for the same state, +// rather than one that reads as a volume already restored. +func TestAReplayRefusesWhenTheCreatorStartsRollingBackUnderItsClaim(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-under") + seedRestoreSnapshot(t, st, "src-under", "snap-under", []string{"node-a"}) + + snap, err := st.Snapshots().Get(ctx, "src-under", "snap-under") + if err != nil { + t.Fatalf("read the snapshot: %v", err) + } + + seedRestoreLeftover(t, st, "src-under", "snap-under", "dst-under", store.WithRestoreMarker(nil, &snap)) + + if err := st.VolumeDefinitions().Create(ctx, "dst-under", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the hydrated volume: %v", err) + } + + base, stop := startServerWithStore(t, rollbackStartsUnderTheClaimStore{Store: st, target: "dst-under"}) + defer stop() + + code, rc := restoreAnswer(t, base, "src-under", "snap-under", map[string]any{"to_resource": "dst-under"}) + if code != http.StatusConflict { + t.Fatalf("replay whose claim met a rollback = %d %q, want 409", code, rc.Message) + } + + if rc.RetCode&apiCallRcFailExistsRscDfn != apiCallRcFailExistsRscDfn { + t.Errorf("refusal ret_code %#x lacks the FAIL_EXISTS_RSC_DFN band", rc.RetCode) + } + + if rc.RetCode&apiCallRcFailExistsVlmDfn == apiCallRcFailExistsVlmDfn { + t.Errorf("refusal ret_code %#x carries the FAIL_EXISTS_VLM_DFN band", rc.RetCode) + } +} + +// The clone's replay refuses the same way when its claim finds the creator +// rolling back: answered 201, the clone would be deleted under the caller. +func TestACloneReplayRefusesWhenTheCreatorStartsRollingBackUnderItsClaim(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedTwoVolumeCloneLeftover(t, st, "src-cunder", "dst-cunder", 0, 1) + + base, stop := startServerWithStore(t, rollbackStartsUnderTheClaimStore{Store: st, target: "dst-cunder"}) + defer stop() + + if code := cloneOnce(t, base, "src-cunder", "dst-cunder", nil); code != http.StatusConflict { + t.Errorf("clone replay whose claim met a rollback = %d, want 409", code) + } +} + +var errMarkRefused = errors.New("apiserver refused the patch") + +// markRefused fails every spec patch of the target. +type markRefused struct { + store.ResourceDefinitionStore + + target string +} + +func (m markRefused) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + if name == m.target { + return errMarkRefused + } + + return m.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) //nolint:wrapcheck // test double +} + +type markRefusedStore struct { + store.Store + + target string +} + +func (s markRefusedStore) ResourceDefinitions() store.ResourceDefinitionStore { + return markRefused{ResourceDefinitionStore: s.Store.ResourceDefinitions(), target: s.target} +} + +// A replay that cannot take the adoption mark does not answer 201: without the +// mark, a creator still rolling back would delete what it answered for. It is +// a retryable 500, and the definition is left as it was. +func TestAReplayThatCannotTakeTheMarkAnswersRetryable(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-nomark") + seedRestoreSnapshot(t, st, "src-nomark", "snap-nomark", []string{"node-a"}) + + snap, err := st.Snapshots().Get(ctx, "src-nomark", "snap-nomark") + if err != nil { + t.Fatalf("read the snapshot: %v", err) + } + + seedRestoreLeftover(t, st, "src-nomark", "snap-nomark", "dst-nomark", store.WithRestoreMarker(nil, &snap)) + + if err := st.VolumeDefinitions().Create(ctx, "dst-nomark", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the hydrated volume: %v", err) + } + + base, stop := startServerWithStore(t, markRefusedStore{Store: st, target: "dst-nomark"}) + defer stop() + + code, rc := restoreAnswer(t, base, "src-nomark", "snap-nomark", map[string]any{"to_resource": "dst-nomark"}) + if code != http.StatusInternalServerError { + t.Fatalf("replay whose mark could not be written = %d %q, want 500", code, rc.Message) + } + + if _, err := st.ResourceDefinitions().Get(ctx, "dst-nomark"); err != nil { + t.Errorf("the definition is gone after a refused replay: %v", err) + } +} diff --git a/pkg/rest/cache_retry.go b/pkg/rest/cache_retry.go index d197901f..25238b1f 100644 --- a/pkg/rest/cache_retry.go +++ b/pkg/rest/cache_retry.go @@ -209,6 +209,11 @@ func getResourceWithCacheRetry(ctx context.Context, st store.Store, rdName, node }) } +// getResourceUncached reads one replica past the cache where the store can. +func getResourceUncached(ctx context.Context, st store.Store, rdName, node string) (apiv1.Resource, error) { + return store.GetResourceUncached(ctx, st, rdName, node) //nolint:wrapcheck // the store wraps with the replica it names +} + // patchNodeSpecWithCacheRetry runs PatchNodeSpec, retrying on // store.ErrNotFound to absorb informer-cache lag. The only caller is // the node-register upsert fall-through: `Nodes().Create` just came diff --git a/pkg/rest/controller_props.go b/pkg/rest/controller_props.go index d235bde2..976d01c8 100644 --- a/pkg/rest/controller_props.go +++ b/pkg/rest/controller_props.go @@ -322,4 +322,5 @@ func applyControllerProps(ctx context.Context, c client.Client, modify *apiv1.Ge func applyControllerPropsModify(props map[string]string, modify *apiv1.GenericPropsModify) { // I1: empty override value deletes the key (set-property KEY ""). applyPropsModify(props, modify.OverrideProps, modify.DeleteProps) + deletePropNamespaces(props, modify.DeleteNamespace) } diff --git a/pkg/rest/delete_namespaces_doors_test.go b/pkg/rest/delete_namespaces_doors_test.go new file mode 100644 index 00000000..7e8d0266 --- /dev/null +++ b/pkg/rest/delete_namespaces_doors_test.go @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "encoding/json" + "net/http" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +func namespacesOnly() apiv1.GenericPropsModify { + return apiv1.GenericPropsModify{DeleteNamespace: []string{"Aux"}} +} + +func wantNamespaceGone(t *testing.T, props map[string]string, door string) { + t.Helper() + + if _, ok := props["Aux/a"]; ok { + t.Errorf("%s: a key under the deleted namespace survived: %v", door, props) + } + + if props["Auxiliary"] != "keep" || props["Other/b"] != "keep" { + t.Errorf("%s: a key outside the namespace went: %v", door, props) + } +} + +func seededProps() map[string]string { + return map[string]string{"Aux/a": "1", "Auxiliary": "keep", "Other/b": "keep"} +} + +// A body carrying only delete_namespaces is a props edit; answered 200 with +// nothing changed, it is the accept-and-drop the field exists to end. +func TestResourceModifyHonoursDeleteNamespaces(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{Name: "rd-ns"}); err != nil { + t.Fatalf("seed the definition: %v", err) + } + + if err := st.Resources().Create(t.Context(), &apiv1.Resource{ + Name: "rd-ns", NodeName: "node-a", Props: seededProps(), + }); err != nil { + t.Fatalf("seed the replica: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(apiv1.ResourceModify{GenericPropsModify: namespacesOnly()}) + + resp := httpPut(t, base+"/v1/resource-definitions/rd-ns/resources/node-a", body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("resource modify = %d, want 200", resp.StatusCode) + } + + res, err := st.Resources().Get(t.Context(), "rd-ns", "node-a") + if err != nil { + t.Fatalf("read the replica: %v", err) + } + + wantNamespaceGone(t, res.Props, "resource modify") +} + +func TestNodeModifyHonoursANamespacesOnlyBody(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + if err := st.Nodes().Create(t.Context(), &apiv1.Node{ + Name: "node-ns", Type: "SATELLITE", Props: seededProps(), + }); err != nil { + t.Fatalf("seed the node: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(apiv1.NodeModify{GenericPropsModify: namespacesOnly()}) + + resp := httpPut(t, base+"/v1/nodes/node-ns", body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("node modify = %d, want 200", resp.StatusCode) + } + + node, err := st.Nodes().Get(t.Context(), "node-ns") + if err != nil { + t.Fatalf("read the node: %v", err) + } + + wantNamespaceGone(t, node.Props, "node modify") +} + +// The controller door patches a ControllerConfig through the API server; the +// merge it re-runs on every retry is checked directly. +func TestControllerPropsModifyHonoursDeleteNamespaces(t *testing.T) { + t.Parallel() + + props := seededProps() + modify := namespacesOnly() + applyControllerPropsModify(props, &modify) + + wantNamespaceGone(t, props, "controller set-property") +} diff --git a/pkg/rest/group_server_owned_props_test.go b/pkg/rest/group_server_owned_props_test.go new file mode 100644 index 00000000..5f153c33 --- /dev/null +++ b/pkg/rest/group_server_owned_props_test.go @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "encoding/json" + "net/http" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// A group that already carries a server-owned prop, written before group +// create and modify refused one, does not hand it to what it spawns: +// linstor-csi fills group props from StorageClass parameters verbatim, so +// inherited, the marker would make every new volume restore another +// resource's snapshot. +func TestSpawnDoesNotInheritServerOwnedGroupProps(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + if err := st.ResourceGroups().Create(t.Context(), &apiv1.ResourceGroup{ + Name: "rg-owned", + Props: map[string]string{ + store.RestoreFromSnapshotProp: "victim:snap", + store.RollbackAbandonedProp: store.RollbackInProgress, + "DrbdOptions/Net/protocol": "C", + }, + }); err != nil { + t.Fatalf("seed the group: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + body, err := json.Marshal(apiv1.ResourceGroupSpawn{ResourceDefinitionName: "pvc-owned"}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + resp := httpPost(t, base+"/v1/resource-groups/rg-owned/spawn", body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("spawn = %d, want 201", resp.StatusCode) + } + + rd, err := st.ResourceDefinitions().Get(t.Context(), "pvc-owned") + if err != nil { + t.Fatalf("read the spawned definition: %v", err) + } + + for _, key := range []string{store.RestoreFromSnapshotProp, store.RollbackAbandonedProp} { + if v, ok := rd.Props[key]; ok { + t.Errorf("spawned definition inherited %s=%q from its group", key, v) + } + } + + if rd.Props["DrbdOptions/Net/protocol"] != "C" { + t.Errorf("spawned definition lost an ordinary group prop: %v", rd.Props) + } +} + +func TestGroupCreateRefusesServerOwnedProps(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + + base, stop := startServerWithStore(t, st) + defer stop() + + for _, rg := range []apiv1.ResourceGroup{ + {Name: "rg-props", Props: map[string]string{store.RestoreFromSnapshotProp: "victim:snap"}}, + {Name: "rg-override", OverrideProps: map[string]string{store.RestoreAdoptedProp: "x"}}, + } { + body, err := json.Marshal(rg) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + resp := httpPost(t, base+"/v1/resource-groups", body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("rg create %s = %d, want 400", rg.Name, resp.StatusCode) + } + + if _, err := st.ResourceGroups().Get(t.Context(), rg.Name); err == nil { + t.Errorf("rg create %s stored the group despite the refusal", rg.Name) + } + } +} + +func TestGroupModifyRefusesSettingServerOwnedProps(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + if err := st.ResourceGroups().Create(t.Context(), &apiv1.ResourceGroup{ + Name: "rg-mod", + Props: map[string]string{store.RestoreFromSnapshotProp: "old:snap"}, + }); err != nil { + t.Fatalf("seed the group: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + set, _ := json.Marshal(apiv1.ResourceGroup{ + OverrideProps: map[string]string{store.RollbackAbandonedProp: store.RollbackInProgress}, + }) + + resp := httpPut(t, base+"/v1/resource-groups/rg-mod", set) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("rg modify setting %s = %d, want 400", store.RollbackAbandonedProp, resp.StatusCode) + } + + // Removing one is how an operator cleans a group written before the + // check, so it stays allowed. + del, _ := json.Marshal(apiv1.ResourceGroup{DeleteProps: []string{store.RestoreFromSnapshotProp}}) + + resp = httpPut(t, base+"/v1/resource-groups/rg-mod", del) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("rg modify removing %s = %d, want 200", store.RestoreFromSnapshotProp, resp.StatusCode) + } + + rg, err := st.ResourceGroups().Get(t.Context(), "rg-mod") + if err != nil { + t.Fatalf("read the group: %v", err) + } + + if _, ok := rg.Props[store.RestoreFromSnapshotProp]; ok { + t.Errorf("the removal did not land: %v", rg.Props) + } + + if _, ok := rg.Props[store.RollbackAbandonedProp]; ok { + t.Errorf("the refused set landed: %v", rg.Props) + } +} + +// Every spelling of a removal cleans a server-owned prop off a group: an +// override to an empty value is the one `rg set-property ` sends. +func TestGroupModifyRemovesServerOwnedPropsInEverySpelling(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + key string + body apiv1.ResourceGroup + }{ + { + name: "delete_props", key: store.RestoreFromSnapshotProp, + body: apiv1.ResourceGroup{DeleteProps: []string{store.RestoreFromSnapshotProp}}, + }, + { + name: "an empty override", key: store.RestoreFromSnapshotProp, + body: apiv1.ResourceGroup{OverrideProps: map[string]string{store.RestoreFromSnapshotProp: ""}}, + }, + // A namespace covers the keys below it, so it reaches the one + // server-owned key that sits in one. + { + name: "delete_namespaces", key: store.RestoreAdoptedProp, + body: apiv1.ResourceGroup{DeleteNamespace: []string{"Blockstor"}}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + if err := st.ResourceGroups().Create(t.Context(), &apiv1.ResourceGroup{ + Name: "rg-clean", + Props: map[string]string{tc.key: "old"}, + }); err != nil { + t.Fatalf("seed the group: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(tc.body) + + resp := httpPut(t, base+"/v1/resource-groups/rg-clean", body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("rg modify removing %s by %s = %d, want 200", tc.key, tc.name, resp.StatusCode) + } + + rg, err := st.ResourceGroups().Get(t.Context(), "rg-clean") + if err != nil { + t.Fatalf("read the group: %v", err) + } + + if _, ok := rg.Props[tc.key]; ok { + t.Errorf("the removal did not land: %v", rg.Props) + } + }) + } +} diff --git a/pkg/rest/kv_store.go b/pkg/rest/kv_store.go index 0e3a7efa..2b6e726b 100644 --- a/pkg/rest/kv_store.go +++ b/pkg/rest/kv_store.go @@ -157,11 +157,12 @@ func handleKVModify(w http.ResponseWriter, r *http.Request) { kvBag[name] = bag } - maps.Copy(bag, patch.OverrideProps) - - for _, key := range patch.DeleteProps { - delete(bag, key) - } + // The same envelope every other props door takes, applied the same way: + // an empty override value deletes the key, and a namespace takes the + // keys below it. + bag = applyPropsModify(bag, patch.OverrideProps, patch.DeleteProps) + deletePropNamespaces(bag, patch.DeleteNamespace) + kvBag[name] = bag kvBagMu.Unlock() diff --git a/pkg/rest/kv_store_test.go b/pkg/rest/kv_store_test.go index fc8da8c6..70c2ff15 100644 --- a/pkg/rest/kv_store_test.go +++ b/pkg/rest/kv_store_test.go @@ -20,6 +20,7 @@ package rest import ( "encoding/json" + "maps" "net/http" "testing" @@ -218,3 +219,36 @@ func TestKVPutDeletePersistInProcessLocalBag(t *testing.T) { t.Errorf("post-delete Props: got %+v, want empty", third[0].Props) } } + +// The key-value-store door takes the props envelope the way every other door +// does: delete_namespaces takes the keys below a namespace and leaves a key +// spelled as it, and an empty override value deletes its key. +func TestKVModifyHonoursTheWholeEnvelope(t *testing.T) { + kvBagMu.Lock() + kvBag["kv-envelope"] = map[string]string{"Aux": "x", "Aux/a": "y", "gone": "z", "keep": "k"} + kvBagMu.Unlock() + + base, stop := startServerWithStore(t, store.NewInMemory()) + defer stop() + + put, _ := json.Marshal(apiv1.GenericPropsModify{ + OverrideProps: map[string]string{"gone": ""}, + DeleteNamespace: []string{"Aux"}, + }) + + resp := httpPut(t, base+"/v1/key-value-store/kv-envelope", put) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("PUT status: got %d, want 200", resp.StatusCode) + } + + kvBagMu.Lock() + got := maps.Clone(kvBag["kv-envelope"]) + kvBagMu.Unlock() + + want := map[string]string{"Aux": "x", "keep": "k"} + if !maps.Equal(got, want) { + t.Errorf("bag after the modify = %v, want %v", got, want) + } +} diff --git a/pkg/rest/leftover_gates_test.go b/pkg/rest/leftover_gates_test.go new file mode 100644 index 00000000..eb4ebbda --- /dev/null +++ b/pkg/rest/leftover_gates_test.go @@ -0,0 +1,799 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "context" + "encoding/json" + "net/http" + "slices" + "strings" + "testing" + + linstor "github.com/LINBIT/golinstor" + lapi "github.com/LINBIT/golinstor/client" + + "github.com/cockroachdb/errors" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +func seedRestoreLeftover(t *testing.T, st store.Store, src, snap, target string, props map[string]string) { + t.Helper() + + all := map[string]string{restoreFromSnapshotKey: restoreMarker(src, snap)} + for k, v := range props { + all[k] = v + } + + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{ + Name: target, Props: all, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } +} + +func restoreAnswer(t *testing.T, base, src, snap string, body map[string]any) (int, apiv1.APICallRc) { + t.Helper() + + raw, err := json.Marshal(body) + if err != nil { + t.Fatalf("marshal restore body: %v", err) + } + + resp := httpPost(t, base+"/v1/resource-definitions/"+src+"/snapshot-restore-resource/"+snap, raw) + defer func() { _ = resp.Body.Close() }() + + var rcs []apiv1.APICallRc + if err := json.NewDecoder(resp.Body).Decode(&rcs); err != nil || len(rcs) == 0 { + t.Fatalf("decode the answer (status %d): %v", resp.StatusCode, err) + } + + // golinstor's ApiCallError.Is matches a band on any rc of the answer, so + // the band a test checks is every rc's, folded into the first. + rc := rcs[0] + for _, more := range rcs[1:] { + rc.RetCode |= more.RetCode + } + + return resp.StatusCode, rc +} + +// The restore resumed a volume-less leftover whose only replica was going +// before anything looked at its replicas: a bare restore answered 201 with a +// volume hydrated into the tear-down, a node-named one hydrated first and was +// stopped only by the stamp. +func TestSnapshotRestoreResumeRefusesAVolumeLessLeftoverBeingTornDown(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + body map[string]any + }{ + {name: "bare", body: map[string]any{"to_resource": "dst-vl-bare"}}, + {name: "node-named", body: map[string]any{"to_resource": "dst-vl-named", "node_names": []string{"node-a"}}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + target, _ := tc.body["to_resource"].(string) + seedDeployedCloneSource(t, st, "src-vl") + seedRestoreSnapshot(t, st, "src-vl", "snap-vl", []string{"node-a"}) + seedRestoreLeftover(t, st, "src-vl", "snap-vl", target, nil) + seedTerminatingReplica(t, st, target, "node-a") + + base, stop := startServerWithStore(t, st) + defer stop() + + code, rc := restoreAnswer(t, base, "src-vl", "snap-vl", tc.body) + if code != http.StatusConflict || !strings.Contains(rc.Message, "torn down") { + t.Errorf("resume over a leftover being torn down = %d %q, want the tear-down refusal", code, rc.Message) + } + + if vds, _ := st.VolumeDefinitions().List(ctx, target); len(vds) > 0 { + t.Errorf("the resume hydrated %d volume(s) into a leftover being torn down", len(vds)) + } + }) + } +} + +// The shape arms returned before the replicas were read, so a multi-volume +// leftover missing a snapshot volume was resumed into its tear-down too. +func TestSnapshotRestoreResumeRefusesAPartialLeftoverBeingTornDown(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-pl") + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "snap-pl", ResourceName: "src-pl", Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{ + {VolumeNumber: 0, SizeKib: 64 * 1024}, {VolumeNumber: 1, SizeKib: 64 * 1024}, + }, + }); err != nil { + t.Fatalf("seed the snapshot: %v", err) + } + + seedRestoreLeftover(t, st, "src-pl", "snap-pl", "dst-pl", nil) + + if err := st.VolumeDefinitions().Create(ctx, "dst-pl", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the first volume: %v", err) + } + + seedTerminatingReplica(t, st, "dst-pl", "node-a") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code, rc := restoreAnswer(t, base, "src-pl", "snap-pl", + map[string]any{"to_resource": "dst-pl"}); code != http.StatusConflict { + t.Errorf("resume over a partial leftover being torn down = %d %q, want 409", code, rc.Message) + } + + if _, err := st.VolumeDefinitions().Get(ctx, "dst-pl", 1); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the missing volume was hydrated into the tear-down: %v", err) + } +} + +// One leftover that is both being torn down and carries an abandoned-rollback +// mark got opposite advice: wait on one door, delete by hand on the other. The +// tear-down refusal is the more precise one, and both doors give it. +func TestTearDownOutranksTheAbandonedRollbackMarkOnBothDoors(t *testing.T) { + t.Parallel() + + mark := map[string]string{rollbackAbandonedKey: "replicas"} + + t.Run("restore", func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-om") + seedRestoreSnapshot(t, st, "src-om", "snap-om", []string{"node-a"}) + seedRestoreLeftover(t, st, "src-om", "snap-om", "dst-om", mark) + seedTerminatingReplica(t, st, "dst-om", "node-a") + + base, stop := startServerWithStore(t, st) + defer stop() + + if _, rc := restoreAnswer(t, base, "src-om", "snap-om", + map[string]any{"to_resource": "dst-om"}); !strings.Contains(rc.Message, "torn down") { + t.Errorf("restore answer %q is not the tear-down refusal", rc.Message) + } + }) + + t.Run("clone", func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-oc") + seedRestoreLeftover(t, st, "src-oc", cloneSnapshotName("dst-oc"), "dst-oc", mark) + seedTerminatingReplica(t, st, "dst-oc", "node-a") + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-oc", map[string]any{"name": "dst-oc"}) + defer func() { _ = resp.Body.Close() }() + + if rc := decodeCloneMessage(t, resp); !strings.Contains(rc.Message, "torn down") { + t.Errorf("clone answer %q is not the tear-down refusal", rc.Message) + } + }) +} + +// replicaReadsMiss answers every cached replica Get with NotFound, the way a +// cache that has not seen a replica's create does, while the replica is there +// and the API server answers for it. +type replicaReadsMiss struct{ store.Store } + +type replicaReadsMissResources struct{ store.ResourceStore } + +func (r replicaReadsMiss) Resources() store.ResourceStore { + return replicaReadsMissResources{r.Store.Resources()} +} + +func (replicaReadsMissResources) Get(_ context.Context, rdName, node string) (apiv1.Resource, error) { + return apiv1.Resource{}, errors.Wrapf(store.ErrNotFound, "resource %s/%s", rdName, node) +} + +func (r replicaReadsMissResources) GetUncached(ctx context.Context, rdName, node string) (apiv1.Resource, error) { + return r.ResourceStore.Get(ctx, rdName, node) //nolint:wrapcheck // pass-through test double +} + +// A replica the Create reports as existing and a lagging cache cannot find was +// taken as a failed materialisation: a fresh restore was rolled back, a resume +// answered 404 for a volume that is fine. The deciding read goes to the API +// server, which has it. +func TestSnapshotRestoreTakesAnUnreadableExistingReplicaAsPlaced(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, backend, "src-rm") + seedRestoreSnapshot(t, backend, "src-rm", "snap-rm", []string{"node-a"}) + + if err := backend.Resources().Create(ctx, &apiv1.Resource{Name: "dst-rm", NodeName: "node-a"}); err != nil { + t.Fatalf("seed the live replica: %v", err) + } + + base, stop := startServerWithStore(t, replicaReadsMiss{backend}) + defer stop() + + if code := restoreOnce(t, base, "src-rm", "snap-rm", map[string]any{ + "to_resource": "dst-rm", "node_names": []string{"node-a"}, + }); code != http.StatusCreated { + t.Errorf("restore over a live replica the cache cannot read yet = %d, want 201", code) + } + + if _, err := backend.ResourceDefinitions().Get(ctx, "dst-rm"); err != nil { + t.Errorf("the restore was rolled back over a replica that is fine: %v", err) + } +} + +// The refusal over a replica still being deleted carries no FAIL_EXISTS_RSC +// band: the replica is being reaped, not made by a concurrent restore, and +// the 409 stays a retryable failure. +func TestSnapshotRestoreRefusalOverADeletingReplicaCarriesNoExistsBand(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-band") + seedRestoreSnapshot(t, st, "src-band", "snap-band", []string{"node-a", "node-b"}) + seedRestoreLeftover(t, st, "src-band", "snap-band", "dst-band", nil) + + if err := st.Resources().Create(t.Context(), &apiv1.Resource{Name: "dst-band", NodeName: "node-b"}); err != nil { + t.Fatalf("seed the live replica: %v", err) + } + + seedTerminatingReplica(t, st, "dst-band", "node-a") + + base, stop := startServerWithStore(t, st) + defer stop() + + code, rc := restoreAnswer(t, base, "src-band", "snap-band", map[string]any{ + "to_resource": "dst-band", "node_names": []string{"node-a"}, + }) + if code != http.StatusConflict { + t.Fatalf("restore stamping over a replica still being deleted = %d %q, want 409", code, rc.Message) + } + + // Asked the way linstor-csi asks it, through golinstor's own match, and + // by this server's own band, which golinstor reads once the numbering + // matches upstream's. + if (&lapi.ApiCallRc{RetCode: rc.RetCode}).Is(linstor.FailExistsRsc) || + rc.RetCode&apiCallRcFailExistsRsc == apiCallRcFailExistsRsc { + t.Errorf("refusal ret_code %#x carries the FAIL_EXISTS_RSC band of a resource that already exists", rc.RetCode) + } +} + +// A diskless replica already on the requested node holds no copy of the data. +// Counted as the restore's own placement, the restore answered 201 with no +// data on any node; it is refused, without the FAIL_EXISTS band linstor-csi +// takes as success. +func TestSnapshotRestoreDoesNotCountADisklessReplicaAsPlaced(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-dl") + seedRestoreSnapshot(t, st, "src-dl", "snap-dl", []string{"node-a"}) + seedRestoreLeftover(t, st, "src-dl", "snap-dl", "dst-dl", nil) + + if err := st.Resources().Create(t.Context(), &apiv1.Resource{ + Name: "dst-dl", NodeName: "node-a", Flags: []string{apiv1.ResourceFlagDiskless}, + }); err != nil { + t.Fatalf("seed the diskless replica: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + code, rc := restoreAnswer(t, base, "src-dl", "snap-dl", map[string]any{ + "to_resource": "dst-dl", "node_names": []string{"node-a"}, + }) + if code != http.StatusConflict { + t.Fatalf("restore onto a node holding a diskless replica = %d %q, want 409", code, rc.Message) + } + + if (&lapi.ApiCallRc{RetCode: rc.RetCode}).Is(linstor.FailExistsRsc) || + rc.RetCode&apiCallRcFailExistsRsc == apiCallRcFailExistsRsc { + t.Errorf("refusal ret_code %#x carries the FAIL_EXISTS_RSC band of a resource that already exists", rc.RetCode) + } +} + +// The controller places a tie-breaker witness once two replicas with a disk +// exist, so a restore placing onto three nodes can meet it on the third. It is +// promoted to the replica the restore asked for, as autoplace promotes one. +func TestSnapshotRestorePromotesATieBreakerOnARequestedNode(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-tb") + seedRestoreSnapshot(t, st, "src-tb", "snap-tb", []string{"node-a"}) + seedRestoreLeftover(t, st, "src-tb", "snap-tb", "dst-tb", nil) + + if err := st.Resources().Create(t.Context(), &apiv1.Resource{ + Name: "dst-tb", NodeName: "node-a", + Flags: []string{apiv1.ResourceFlagDiskless, apiv1.ResourceFlagTieBreaker}, + }); err != nil { + t.Fatalf("seed the witness: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + code, rc := restoreAnswer(t, base, "src-tb", "snap-tb", map[string]any{ + "to_resource": "dst-tb", "node_names": []string{"node-a"}, + }) + if code != http.StatusCreated { + t.Fatalf("restore onto a node holding the witness = %d %q, want 201", code, rc.Message) + } + + res, err := st.Resources().Get(t.Context(), "dst-tb", "node-a") + if err != nil { + t.Fatalf("read the replica: %v", err) + } + + if !store.HoldsData(&res) { + t.Errorf("the witness was counted as placed without a disk: flags %v", res.Flags) + } +} + +// A witness whose promotion finds no pool to take stays diskless, and is then +// refused like any replica without a disk rather than counted as placed. +func TestSnapshotRestoreRefusesAWitnessItCouldNotGiveADisk(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-nopool") + + src, err := st.Resources().Get(ctx, "src-nopool", "node-a") + if err != nil { + t.Fatalf("read the source replica: %v", err) + } + + src.Props = nil + if err := st.Resources().Update(ctx, &src); err != nil { + t.Fatalf("leave the source without a recorded pool: %v", err) + } + + seedRestoreSnapshot(t, st, "src-nopool", "snap-nopool", []string{"node-a"}) + seedRestoreLeftover(t, st, "src-nopool", "snap-nopool", "dst-nopool", nil) + + if err := st.Resources().Create(ctx, &apiv1.Resource{ + Name: "dst-nopool", NodeName: "node-a", + Flags: []string{apiv1.ResourceFlagDiskless, apiv1.ResourceFlagTieBreaker}, + }); err != nil { + t.Fatalf("seed the witness: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + code, rc := restoreAnswer(t, base, "src-nopool", "snap-nopool", map[string]any{ + "to_resource": "dst-nopool", "node_names": []string{"node-a"}, + }) + if code != http.StatusConflict { + t.Errorf("restore onto a witness no pool could be found for = %d %q, want 409", code, rc.Message) + } + + // The refusal leaves the witness as it was: stripped of TIE_BREAKER it + // would read as an operator's diskless replica on every retry. + witness, err := st.Resources().Get(ctx, "dst-nopool", "node-a") + if err != nil { + t.Fatalf("read the witness: %v", err) + } + + if !slices.Contains(witness.Flags, apiv1.ResourceFlagTieBreaker) { + t.Errorf("the refused witness lost its TIE_BREAKER flag: %v", witness.Flags) + } +} + +// A witness whose promotion fails is not counted as placed: the restore is +// refused, without the FAIL_EXISTS band, and the witness keeps its flags. +func TestSnapshotRestoreRefusesAWitnessItFailedToPromote(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-tbp") + seedRestoreSnapshot(t, st, "src-tbp", "snap-tbp", []string{"node-a"}) + seedRestoreLeftover(t, st, "src-tbp", "snap-tbp", "dst-tbp", nil) + + if err := st.Resources().Create(ctx, &apiv1.Resource{ + Name: "dst-tbp", NodeName: "node-a", + Flags: []string{apiv1.ResourceFlagDiskless, apiv1.ResourceFlagTieBreaker}, + }); err != nil { + t.Fatalf("seed the witness: %v", err) + } + + base, stop := startServerWithStore(t, faultyStore{ + Store: st, f: storeFaults{target: "dst-tbp", replicaPatch: errors.New("apiserver refused the patch")}, + }) + defer stop() + + code, rc := restoreAnswer(t, base, "src-tbp", "snap-tbp", map[string]any{ + "to_resource": "dst-tbp", "node_names": []string{"node-a"}, + }) + if code != http.StatusInternalServerError { + t.Fatalf("restore over a witness whose promotion failed = %d %q, want 500", code, rc.Message) + } + + if (&lapi.ApiCallRc{RetCode: rc.RetCode}).Is(linstor.FailExistsRsc) || + rc.RetCode&apiCallRcFailExistsRsc == apiCallRcFailExistsRsc { + t.Errorf("refusal ret_code %#x carries the FAIL_EXISTS_RSC band of a resource that already exists", rc.RetCode) + } + + witness, err := st.Resources().Get(ctx, "dst-tbp", "node-a") + if err != nil { + t.Fatalf("read the witness: %v", err) + } + + if !slices.Contains(witness.Flags, apiv1.ResourceFlagTieBreaker) { + t.Errorf("the witness lost its TIE_BREAKER flag: %v", witness.Flags) + } +} + +// The authoritative read finding nothing means the cached read that found +// the leftover was stale: there is no mark to refuse over. +func TestAbandonedRollbackGateProceedsWhenTheDefinitionIsGone(t *testing.T) { + t.Parallel() + + s := &Server{Store: store.NewInMemory()} + + if status, refusal := s.abandonedRollbackRefusal(t.Context(), "restore", "dst-gone"); refusal != nil { + t.Errorf("gate over a definition that is gone = %d %q, want no refusal", status, refusal.Message) + } +} + +func seedTerminatingReplica(t *testing.T, st store.Store, rdName, node string) { + t.Helper() + + if err := st.Resources().Create(t.Context(), &apiv1.Resource{ + Name: rdName, NodeName: node, Flags: []string{apiv1.ResourceFlagDelete}, + }); err != nil { + t.Fatalf("seed the terminating replica: %v", err) + } +} + +// A volume-less leftover was judged unfinished before its replicas were looked +// at, and the resume then counted a replica mid-tear-down as placed: 201 over +// a definition with no live replica. +func TestRDCloneResumeRefusesAVolumeLessLeftoverWhoseReplicaIsGoing(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-na") + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-na", + Props: map[string]string{restoreFromSnapshotKey: restoreMarker("src-na", cloneSnapshotName("dst-na"))}, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + seedTerminatingReplica(t, st, "dst-na", "node-a") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-na", "dst-na", nil); code == http.StatusCreated { + t.Errorf("resume over a leftover whose only replica is being torn down = 201") + } + + // Refused before the resume writes anything: hydrating volumes into a + // definition its tear-down is reaping races that tear-down. + if vds, _ := st.VolumeDefinitions().List(ctx, "dst-na"); len(vds) > 0 { + t.Errorf("the resume ran over a leftover being torn down: %d volume(s) hydrated", len(vds)) + } +} + +// A fresh clone under a name whose old replica is still held by its finalizer +// met AlreadyExists on the stamp, took it for its own placement, and reported +// the clone done over a definition with no live replica. +func TestRDCloneDoesNotCountATerminatingReplicaAsPlaced(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-nb") + seedTerminatingReplica(t, st, "dst-nb", "node-a") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-nb", "dst-nb", nil); code == http.StatusCreated { + t.Errorf("clone over a replica still being torn down = 201") + } + + if _, err := st.ResourceDefinitions().Get(ctx, "dst-nb"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the clone this request created was not rolled back: %v", err) + } +} + +// The restore writes the abandoned-rollback mark through the same failed +// materialisation the clone does, and resumed the leftover its twin refuses. +func TestSnapshotRestoreResumeRefusesALeftoverWhoseRollbackGaveUp(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-rgu") + seedRestoreSnapshot(t, st, "src-rgu", "snap-rgu", []string{"node-a"}) + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-rgu", + Props: map[string]string{ + restoreFromSnapshotKey: restoreMarker("src-rgu", "snap-rgu"), + rollbackAbandonedKey: "replicas", + }, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := restoreOnce(t, base, "src-rgu", "snap-rgu", + map[string]any{"to_resource": "dst-rgu"}); code != http.StatusConflict { + t.Errorf("restore resume over a leftover whose rollback gave up = %d, want 409", code) + } + + if vds, _ := st.VolumeDefinitions().List(ctx, "dst-rgu"); len(vds) > 0 { + t.Errorf("the resume ran over the marked leftover: %d volume(s) hydrated", len(vds)) + } +} + +var errReplicaWriteRefused = errors.New("replica write refused") + +// replicaCreateFails refuses every replica write, the way an apiserver that +// times out on the stamp does. +type replicaCreateFails struct{ store.ResourceStore } + +func (replicaCreateFails) Create(context.Context, *apiv1.Resource) error { + return errReplicaWriteRefused +} + +type replicaCreateFailsStore struct{ store.Store } + +func (r replicaCreateFailsStore) Resources() store.ResourceStore { + return replicaCreateFails{r.Store.Resources()} +} + +// The restore resumes a leftover an earlier attempt left. A failure while it +// does is not this request's partial work to undo: the definition was there +// before it, and rolling it back takes another attempt's work with it. +func TestSnapshotRestoreResumeFailureLeavesTheAdoptedLeftover(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, backend, "src-adopt") + seedRestoreSnapshot(t, backend, "src-adopt", "snap-adopt", []string{"node-a"}) + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-adopt", + Props: map[string]string{restoreFromSnapshotKey: restoreMarker("src-adopt", "snap-adopt")}, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + base, stop := startServerWithStore(t, replicaCreateFailsStore{backend}) + defer stop() + + code := restoreOnce(t, base, "src-adopt", "snap-adopt", map[string]any{ + "to_resource": "dst-adopt", "node_names": []string{"node-a"}, + }) + if code == http.StatusCreated { + t.Fatalf("resume with every replica write refused = 201") + } + + if _, err := backend.ResourceDefinitions().Get(ctx, "dst-adopt"); err != nil { + t.Errorf("the adopted leftover was rolled back by a request that did not create it: %v", err) + } +} + +// A leftover whose rollback gave up may hold less than the clone intended, +// and finishing it is exactly what the mark exists to stop. The resume is +// refused as the replay of a finished one is. +func TestRDCloneResumeRefusesALeftoverWhoseRollbackGaveUp(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-gaveup") + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-gaveup", + Props: map[string]string{ + restoreFromSnapshotKey: restoreMarker("src-gaveup", cloneSnapshotName("dst-gaveup")), + rollbackAbandonedKey: "replicas", + }, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-gaveup", "dst-gaveup", nil); code != http.StatusConflict { + t.Errorf("resume over a leftover whose rollback gave up = %d, want 409", code) + } + + if vds, _ := st.VolumeDefinitions().List(ctx, "dst-gaveup"); len(vds) > 0 { + t.Errorf("the resume ran over the marked leftover: %d volume(s) hydrated", len(vds)) + } +} + +// The replay of a finished restore answers for a leftover the same way the +// resume does: one parented to a group that is gone is refused, not reported +// restored. +func TestSnapshotRestoreReplayOfAFinishedLeftoverChecksItsGroup(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-fin") + seedRestoreSnapshot(t, st, "src-fin", "snap-fin", []string{"node-a"}) + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-fin", + ResourceGroupName: "grp-gone", + Props: map[string]string{restoreFromSnapshotKey: restoreMarker("src-fin", "snap-fin")}, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + if err := st.VolumeDefinitions().Create(ctx, "dst-fin", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the leftover's volume: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + code := restoreOnce(t, base, "src-fin", "snap-fin", map[string]any{"to_resource": "dst-fin"}) + if code == http.StatusCreated { + t.Errorf("replay of a finished restore parented to a group that is gone = 201") + } +} + +// A rollback that has not reported a step may still be running, and ends on its +// own. The replay gate read the in-progress mark as an unknown step and told +// the operator the compensation could not complete and to delete by hand. +func TestRDCloneResumeWordsARollbackStillInProgress(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-rolling") + + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{ + Name: "dst-rolling", + Props: map[string]string{ + restoreFromSnapshotKey: restoreMarker("src-rolling", cloneSnapshotName("dst-rolling")), + rollbackAbandonedKey: rollbackInProgress, + }, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-rolling", map[string]any{"name": "dst-rolling"}) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("resume over a rollback in progress = %d, want 409", resp.StatusCode) + } + + rc := decodeCloneMessage(t, resp) + if !strings.Contains(rc.Correc, "once that rollback has finished") { + t.Errorf("correction %q does not say the rollback may still finish", rc.Correc) + } +} + +// The clone door gives the same typed answer the restore door does when the +// replica it stamps is still being deleted, instead of a bare 500. +func TestRDCloneRefusalOverADeletingReplicaCarriesNoExistsBand(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-cband") + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-cband", + Props: map[string]string{restoreFromSnapshotKey: restoreMarker("src-cband", cloneSnapshotName("dst-cband"))}, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + if err := st.Resources().Create(ctx, &apiv1.Resource{Name: "dst-cband", NodeName: "node-b"}); err != nil { + t.Fatalf("seed the live replica: %v", err) + } + + seedTerminatingReplica(t, st, "dst-cband", "node-a") + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-cband", map[string]any{"name": "dst-cband"}) + defer func() { _ = resp.Body.Close() }() + + rc := decodeCloneMessage(t, resp) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("clone stamping over a replica still being deleted = %d %q, want 409", resp.StatusCode, rc.Message) + } + + // Asked the way linstor-csi asks it, through golinstor's own match, and + // by this server's own band, which golinstor reads once the numbering + // matches upstream's. + if (&lapi.ApiCallRc{RetCode: rc.RetCode}).Is(linstor.FailExistsRsc) || + rc.RetCode&apiCallRcFailExistsRsc == apiCallRcFailExistsRsc { + t.Errorf("refusal ret_code %#x carries the FAIL_EXISTS_RSC band of a resource that already exists", rc.RetCode) + } +} + +// A witness the restore promoted is the restore's replica, and a rollback of +// that restore deletes it like one it created. +func TestAPromotedWitnessIsCountedAsPlaced(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-tbc") + seedRestoreSnapshot(t, st, "src-tbc", "snap-tbc", []string{"node-a"}) + seedRestoreLeftover(t, st, "src-tbc", "snap-tbc", "dst-tbc", nil) + + if err := st.Resources().Create(t.Context(), &apiv1.Resource{ + Name: "dst-tbc", NodeName: "node-a", + Flags: []string{apiv1.ResourceFlagDiskless, apiv1.ResourceFlagTieBreaker}, + }); err != nil { + t.Fatalf("seed the witness: %v", err) + } + + created, err := (&Server{Store: st}).stampRestoredReplica(t.Context(), &apiv1.Resource{ + Name: "dst-tbc", NodeName: "node-a", Props: map[string]string{"StorPoolName": "zfs-thin"}, + }) + if err != nil { + t.Fatalf("stamp over the witness: %v", err) + } + + if !created { + t.Error("the promoted witness was not counted as placed, so a rollback would leave it behind") + } +} + +// A replica with a disk already on the node is a previous attempt's, not this +// call's: counted as placed, a rollback of this call would delete it. +func TestAnExistingReplicaIsNotCountedAsPlaced(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-ex") + seedRestoreSnapshot(t, st, "src-ex", "snap-ex", []string{"node-a"}) + seedRestoreLeftover(t, st, "src-ex", "snap-ex", "dst-ex", nil) + + if err := st.Resources().Create(t.Context(), &apiv1.Resource{ + Name: "dst-ex", NodeName: "node-a", Props: map[string]string{"StorPoolName": "zfs-thin"}, + }); err != nil { + t.Fatalf("seed the earlier attempt's replica: %v", err) + } + + created, err := (&Server{Store: st}).stampRestoredReplica(t.Context(), &apiv1.Resource{ + Name: "dst-ex", NodeName: "node-a", Props: map[string]string{"StorPoolName": "zfs-thin"}, + }) + if err != nil { + t.Fatalf("stamp over the existing replica: %v", err) + } + + if created { + t.Error("an existing replica was counted as this call's, so a rollback would delete it") + } +} diff --git a/pkg/rest/materialise_refusal_test.go b/pkg/rest/materialise_refusal_test.go new file mode 100644 index 00000000..0abedc54 --- /dev/null +++ b/pkg/rest/materialise_refusal_test.go @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "context" + "maps" + "net/http" + "sync/atomic" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// rollbackStartsOnAdoption plays the attempt that created the definition +// starting its rollback just before this request adopts it: the in-progress +// rollback mark lands first, so the adopting patch finds it and writes nothing. +// A patch is the adopting one when it would write the adoption mark. +type rollbackStartsOnAdoption struct{ store.ResourceDefinitionStore } + +func (r rollbackStartsOnAdoption) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + current, err := r.Get(ctx, name) + if err == nil { + probe := current + probe.Props = maps.Clone(current.Props) + + if probe.Props == nil { + probe.Props = map[string]string{} + } + + if mutate(&probe) == nil && probe.Props[store.RestoreAdoptedProp] != current.Props[store.RestoreAdoptedProp] { + _ = r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, + func(rd *apiv1.ResourceDefinition) error { + rd.Props[store.RollbackAbandonedProp] = store.RollbackInProgress + + return nil + }) + } + } + + return r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) //nolint:wrapcheck // test double +} + +type rollbackStartsOnAdoptionStore struct{ store.Store } + +func (r rollbackStartsOnAdoptionStore) ResourceDefinitions() store.ResourceDefinitionStore { + return rollbackStartsOnAdoption{r.Store.ResourceDefinitions()} +} + +// Both doors answer a creator already rolling back with the typed 409 of the +// definition it is about, and write nothing into the leftover. +func TestResumeOverACreatorRollingBackCarriesTheDefinitionBand(t *testing.T) { + t.Parallel() + + t.Run("restore", func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-rb") + seedRestoreSnapshot(t, backend, "src-rb", "snap-rb", []string{"node-a"}) + seedRestoreLeftover(t, backend, "src-rb", "snap-rb", "dst-rb", nil) + + base, stop := startServerWithStore(t, rollbackStartsOnAdoptionStore{backend}) + defer stop() + + code, rc := restoreAnswer(t, base, "src-rb", "snap-rb", map[string]any{"to_resource": "dst-rb"}) + if code != http.StatusConflict { + t.Fatalf("restore resume over a creator rolling back = %d %q, want 409", code, rc.Message) + } + + if rc.RetCode&apiCallRcFailExistsRscDfn != apiCallRcFailExistsRscDfn { + t.Errorf("refusal ret_code %#x carries no FAIL_EXISTS_RSC_DFN band", rc.RetCode) + } + + if vds, _ := backend.VolumeDefinitions().List(t.Context(), "dst-rb"); len(vds) != 0 { + t.Errorf("the refused resume hydrated %d volume(s)", len(vds)) + } + }) + + t.Run("clone", func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-cb") + + if err := backend.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{ + Name: "dst-cb", + Props: map[string]string{restoreFromSnapshotKey: restoreMarker("src-cb", cloneSnapshotName("dst-cb"))}, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + base, stop := startServerWithStore(t, rollbackStartsOnAdoptionStore{backend}) + defer stop() + + resp := postClone(t, base, "src-cb", map[string]any{"name": "dst-cb"}) + defer func() { _ = resp.Body.Close() }() + + rc := decodeCloneMessage(t, resp) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("clone resume over a creator rolling back = %d %q, want 409", resp.StatusCode, rc.Message) + } + + if rc.RetCode&apiCallRcFailExistsRscDfn != apiCallRcFailExistsRscDfn { + t.Errorf("refusal ret_code %#x carries no FAIL_EXISTS_RSC_DFN band", rc.RetCode) + } + + if vds, _ := backend.VolumeDefinitions().List(t.Context(), "dst-cb"); len(vds) != 0 { + t.Errorf("the refused resume hydrated %d volume(s)", len(vds)) + } + }) +} + +// sourceGoneAfterSnapshot answers NotFound for the source once the clone has +// taken its internal snapshot: a source deleted while the clone runs, after +// every check the handler makes on it up front. +type sourceGoneAfterSnapshot struct { + store.ResourceDefinitionStore + + backend store.Store + source string + snapshot string + reads *atomic.Int32 +} + +func (s sourceGoneAfterSnapshot) Get(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + if name == s.source { + if _, err := s.backend.Snapshots().Get(ctx, s.source, s.snapshot); err == nil { + s.reads.Add(1) + + return apiv1.ResourceDefinition{}, store.ErrNotFound + } + } + + return s.ResourceDefinitionStore.Get(ctx, name) //nolint:wrapcheck // pass-through test double +} + +type sourceGoneAfterSnapshotStore struct { + store.Store + + source string + snapshot string + reads *atomic.Int32 +} + +func (s sourceGoneAfterSnapshotStore) ResourceDefinitions() store.ResourceDefinitionStore { + return sourceGoneAfterSnapshot{ + ResourceDefinitionStore: s.Store.ResourceDefinitions(), + backend: s.Store, source: s.source, snapshot: s.snapshot, reads: s.reads, + } +} + +// A failure that is not this request's partial work keeps its typed store +// status on the clone door, the way it does on the restore door, rather than a +// bare 500, and leaves no target behind. The internal snapshot stays, as it +// did on every failure before: its name is the target's, so a retry takes it +// up again rather than taking another. +func TestRDCloneAnswersASourceGoneMidCloneWithItsTypedStatus(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-mid") + + var reads atomic.Int32 + + base, stop := startServerWithStore(t, sourceGoneAfterSnapshotStore{ + Store: backend, source: "src-mid", snapshot: cloneSnapshotName("dst-mid"), reads: &reads, + }) + defer stop() + + resp := postClone(t, base, "src-mid", map[string]any{"name": "dst-mid"}) + defer func() { _ = resp.Body.Close() }() + + rc := decodeCloneMessage(t, resp) + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("clone whose source went mid-clone = %d %q, want 404", resp.StatusCode, rc.Message) + } + + if _, err := backend.ResourceDefinitions().Get(t.Context(), "dst-mid"); err == nil { + t.Error("the clone left its target behind") + } + + if reads.Load() == 0 { + t.Fatal("fixture: the source never went away mid-clone") + } + + snaps, _ := backend.Snapshots().ListByDefinition(t.Context(), "src-mid") + if len(snaps) != 1 || snaps[0].Name != cloneSnapshotName("dst-mid") { + t.Errorf("snapshots on the source = %v, want only the internal one a retry takes up", snaps) + } +} diff --git a/pkg/rest/node_connections.go b/pkg/rest/node_connections.go index dde2941f..bb4276ca 100644 --- a/pkg/rest/node_connections.go +++ b/pkg/rest/node_connections.go @@ -458,18 +458,7 @@ func mergeNodeConnectionPropsInto( // I1: empty override value deletes the key (set-property KEY ""). applyPropsModify(props, modify.OverrideProps, modify.DeleteProps) - // Namespace-prefix delete (e.g. `delete_namespaces: ["DrbdOptions/Net"]`) - // removes every key under the given prefix. Matches upstream - // LINSTOR's `GenericPropsModify.delete_namespaces` semantic. - for _, ns := range modify.DeleteNamespace { - prefix := strings.TrimSuffix(ns, "/") + "/" - - for k := range props { - if strings.HasPrefix(k, prefix) || k == ns { - delete(props, k) - } - } - } + deletePropNamespaces(props, modify.DeleteNamespace) if len(props) == 0 { delete(conn, key) diff --git a/pkg/rest/nodes.go b/pkg/rest/nodes.go index 2c3354ad..ec569aa0 100644 --- a/pkg/rest/nodes.go +++ b/pkg/rest/nodes.go @@ -945,11 +945,14 @@ func (s *Server) handleNodeUpdate(w http.ResponseWriter, r *http.Request) { } } - if len(patch.OverrideProps) > 0 || len(patch.DeleteProps) > 0 { + // delete_namespaces alone is a props edit too: gated on the first two + // halves only, a namespaces-only body never reached the patch. + if len(patch.OverrideProps) > 0 || len(patch.DeleteProps) > 0 || len(patch.DeleteNamespace) > 0 { err = s.Store.Nodes().PatchProps(r.Context(), name, func(props map[string]string) error { // I1: route through the shared core so an empty // override value deletes the key (set-property KEY ""). applyPropsModify(props, patch.OverrideProps, patch.DeleteProps) + deletePropNamespaces(props, patch.DeleteNamespace) return nil }) diff --git a/pkg/rest/nodes_test.go b/pkg/rest/nodes_test.go index 22cc0c48..9a24cd37 100644 --- a/pkg/rest/nodes_test.go +++ b/pkg/rest/nodes_test.go @@ -1151,6 +1151,12 @@ func newClient(t *testing.T, base string) *lapi.Client { return c } +// pythonLinstorTestUserAgent is what python-linstor 1.27.1 sends +// (linstorapi.py: "PythonLinstor/{v} (API{a})"). The suite's raw requests +// stand in for the CLI, so they name themselves the way it does; the clone +// door answers python-linstor in its own envelope. +const pythonLinstorTestUserAgent = "PythonLinstor/1.27.1 (API1.0.4)" + func httpPost(t *testing.T, addr string, body []byte) *http.Response { t.Helper() @@ -1159,6 +1165,7 @@ func httpPost(t *testing.T, addr string, body []byte) *http.Response { t.Fatalf("new request: %v", err) } req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", pythonLinstorTestUserAgent) resp, err := http.DefaultClient.Do(req) if err != nil { diff --git a/pkg/rest/props_delete_namespaces_test.go b/pkg/rest/props_delete_namespaces_test.go index b0d03597..c77c3822 100644 --- a/pkg/rest/props_delete_namespaces_test.go +++ b/pkg/rest/props_delete_namespaces_test.go @@ -11,6 +11,64 @@ import ( "github.com/cozystack/blockstor/pkg/store" ) +// delete_namespaces is honoured on the path CSI actually takes, not only on +// the volume-less shortcut the first test for it exercised. +func TestRDCloneHonoursDeleteNamespacesOnTheDataPath(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-ns") + + src, err := st.ResourceDefinitions().Get(ctx, "src-ns") + if err != nil { + t.Fatalf("read the seeded source: %v", err) + } + + src.Props = map[string]string{ + "DrbdOptions": "bare", + "DrbdOptions/Net/protocol": "C", + "DrbdOptionsOther": "keep", + } + + if err := st.ResourceDefinitions().Update(ctx, &src); err != nil { + t.Fatalf("put props on the source: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-ns", map[string]any{ + "name": "dst-ns", + "delete_namespaces": []string{"DrbdOptions"}, + "use_zfs_clone": true, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status = %d, want 201", resp.StatusCode) + } + + got, err := st.ResourceDefinitions().Get(ctx, "dst-ns") + if err != nil { + t.Fatalf("get the clone: %v", err) + } + + for _, key := range []string{"DrbdOptions/Net/protocol"} { + if _, present := got.Props[key]; present { + t.Errorf("prop %q survived the namespace delete on the data path", key) + } + } + + // A namespace covers the keys below it; the key spelled as it and one + // that merely starts like it are outside, as upstream deletes them. + for _, key := range []string{"DrbdOptions", "DrbdOptionsOther"} { + if _, present := got.Props[key]; !present { + t.Errorf("prop %q outside the named namespace was deleted", key) + } + } +} + // The modify body declares delete_namespaces and the merge dropped it, so a // modify carrying it answered 200 and changed nothing. func TestRDModifyHonoursDeleteNamespaces(t *testing.T) { @@ -47,7 +105,7 @@ func TestRDModifyHonoursDeleteNamespaces(t *testing.T) { t.Fatalf("read the RD back: %v", err) } - for _, key := range []string{"DrbdOptions", "DrbdOptions/Net/protocol"} { + for _, key := range []string{"DrbdOptions/Net/protocol"} { if _, present := got.Props[key]; present { t.Errorf("prop %q survived delete_namespaces", key) } @@ -55,6 +113,10 @@ func TestRDModifyHonoursDeleteNamespaces(t *testing.T) { // The neighbouring key that merely shares a prefix is not in the // namespace and must stay. + if got.Props["DrbdOptions"] != "top" { + t.Errorf("DrbdOptions = %q, want top: the key spelled as the namespace is outside it", got.Props["DrbdOptions"]) + } + if got.Props["DrbdOptionsOther"] != "keep-me" { t.Errorf("DrbdOptionsOther = %q, want keep-me", got.Props["DrbdOptionsOther"]) } @@ -96,12 +158,16 @@ func TestRGModifyHonoursDeleteNamespaces(t *testing.T) { t.Fatalf("read the RG back: %v", err) } - for _, key := range []string{"DrbdOptions", "DrbdOptions/Net/protocol"} { + for _, key := range []string{"DrbdOptions/Net/protocol"} { if _, present := got.Props[key]; present { t.Errorf("prop %q survived delete_namespaces", key) } } + if got.Props["DrbdOptions"] != "top" { + t.Errorf("DrbdOptions = %q, want top: the key spelled as the namespace is outside it", got.Props["DrbdOptions"]) + } + if got.Props["DrbdOptionsOther"] != "keep-me" { t.Errorf("DrbdOptionsOther = %q, want keep-me", got.Props["DrbdOptionsOther"]) } diff --git a/pkg/rest/props_modify.go b/pkg/rest/props_modify.go index 014d33a1..6697f4c4 100644 --- a/pkg/rest/props_modify.go +++ b/pkg/rest/props_modify.go @@ -18,6 +18,8 @@ limitations under the License. package rest +import "github.com/cozystack/blockstor/pkg/store" + // applyPropsModify is the shared property-bag merge core. Every // LINSTOR object that accepts a `GenericPropsModify` envelope // (override_props + delete_props) applies the same semantic, and @@ -67,15 +69,16 @@ func applyPropsModify(props, override map[string]string, del []string) map[strin } // deletePropNamespaces strips every property under the named namespaces, the -// `delete_namespaces` half of upstream's props-modify envelope. A namespace -// covers the key that spells it exactly and every key below it — `DrbdOptions` -// takes `DrbdOptions` and `DrbdOptions/Net/protocol` with it, and leaves -// `DrbdOptionsOther` alone, because the separator has to be there for a key to -// be inside the namespace rather than merely to start like it. +// `delete_namespaces` half of upstream's props-modify envelope, and is the one +// implementation every door uses. A namespace covers the keys below it and not +// a key that spells it exactly: `DrbdOptions` takes `DrbdOptions/Net/protocol` +// and leaves both `DrbdOptions` and `DrbdOptionsOther` alone. That is +// upstream's reading: its PropsContainer keeps a key and the namespace of the +// same name apart, and removeNamespace clears only the namespace. func deletePropNamespaces(props map[string]string, namespaces []string) { for _, ns := range namespaces { for key := range props { - if key == ns || (len(key) > len(ns) && key[:len(ns)] == ns && key[len(ns)] == '/') { + if store.NamespaceCovers(ns, key) { delete(props, key) } } diff --git a/pkg/rest/rd_clone.go b/pkg/rest/rd_clone.go index c7e105f0..f85d78c7 100644 --- a/pkg/rest/rd_clone.go +++ b/pkg/rest/rd_clone.go @@ -22,14 +22,16 @@ import ( "context" "maps" "net/http" + "slices" + "strconv" "strings" - "time" "github.com/LINBIT/golinstor/client" "github.com/LINBIT/golinstor/clonestatus" "github.com/cockroachdb/errors" apiv1 "github.com/cozystack/blockstor/pkg/api/v1" "github.com/cozystack/blockstor/pkg/store" + "github.com/cozystack/blockstor/pkg/validate" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -77,11 +79,39 @@ import ( // lands on the same snapshot-clone path — an accepted // divergence documented in docs/cli-parity-known-deltas.md. type rdCloneRequest struct { - Name string `json:"name"` - OverrideProps map[string]string `json:"override_props,omitempty"` - DeleteProps []string `json:"delete_props,omitempty"` - SrcSnapName string `json:"src_snap_name,omitempty"` - UseZfsClone bool `json:"use_zfs_clone,omitempty"` + // The props-modify triple every upstream endpoint that edits + // properties carries — override_props, delete_props, + // delete_namespaces. Embedded from golinstor rather than respelled + // field by field, because a respelling is how `delete_namespaces` + // went missing in the first place: a golinstor client filling the + // whole triple would have hit the very 400 that declaring its two + // neighbours was meant to end. golinstor tags it omitempty and + // linstor-csi never fills it, so no CSI clone ever carried it. The + // python CLI's clone verb has no flag for it either — + // it offers --external-name, --use-zfs-clone, --volume-passphrase, + // --layer-list and --resource-group — so this is a wire-level field, + // which is exactly why the decoder is the only thing guarding it. + client.GenericPropsModify + + Name string `json:"name"` + SrcSnapName string `json:"src_snap_name,omitempty"` + UseZfsClone bool `json:"use_zfs_clone,omitempty"` + + // The remaining fields golinstor puts on the wire for this + // endpoint. They are declared because the body is decoded with + // DisallowUnknownFields: a field missing from this struct is a 400 + // before any of the handler runs, whatever its value. + // + // That is not theoretical. linstor-csi defaults LayerList to + // [drbd, storage] in pkg/volume/parameter.go and never sends it + // empty, so every CSI clone-from-volume was refused outright with + // `unknown field "layer_list"` — no StorageClass could avoid it, + // and on Cozystack the platform-wide `cloneStrategyOverride: + // csi-clone` routes every disk clone through here. + LayerList []string `json:"layer_list,omitempty"` + ResourceGroup string `json:"resource_group,omitempty"` + ExternalName string `json:"external_name,omitempty"` + VolumePassphrases []string `json:"volume_passphrases,omitempty"` } // registerRDClone wires the /v1/resource-definitions/{rd}/clone endpoints. @@ -98,6 +128,246 @@ func (s *Server) registerRDClone(mux *http.ServeMux) { s.requireStore(s.handleRDCloneStatus)) } +// storedDefinitionName is the name a definition already under name is stored +// with, or name when there is none. LINSTOR folds the case of a definition's +// name, and the store finds the definition under any spelling, but a +// definition's replicas are selected by the name exactly as stored: judged +// under another spelling, a finished clone or restore counted no replica, +// read as unfinished, and was re-placed onto a node the operator had emptied. +// +// A read that fails is returned, and the caller refuses: kept under the +// request's spelling, a blip that cleared by the next read let the gates +// after this one judge the definition under the wrong name. +func (s *Server) storedDefinitionName(ctx context.Context, name string) (string, error) { + existing, err := s.Store.ResourceDefinitions().Get(ctx, name) + if errors.Is(err, store.ErrNotFound) { + return name, nil + } + + if err != nil { + return name, err //nolint:wrapcheck // answered as the unreadable-target refusal + } + + return existing.Name, nil +} + +// rebindCloneTarget names the clone's target as it is stored; see +// storedDefinitionName. False means a refusal has been written. +func (s *Server) rebindCloneTarget( + ctx context.Context, w http.ResponseWriter, srcName string, req *rdCloneRequest, +) bool { + stored, err := s.storedDefinitionName(ctx, req.Name) + if err != nil { + writeCloneTargetUnreadable(w, srcName, req.Name, err) + + return false + } + + req.Name = stored + + return true +} + +// cloneRequestIsAnswerable refuses, from the body alone, a clone this endpoint +// cannot carry out. False means a refusal has been written. +func cloneRequestIsAnswerable(w http.ResponseWriter, srcName string, req *rdCloneRequest) bool { + // Asked before the name: python-linstor drops `name` from the body when + // it sends external_name, so a missing-name refusal would point the + // operator at the one field they did give. + if !cloneRequestDropsNothing(w, srcName, req) || !cloneTargetNameIsUsable(w, srcName, req) { + return false + } + + // Bug 239: clone-from-an-OPERATOR-NAMED-snapshot is not wired. + // The Bug 232 decoder accepts `src_snap_name` so the CLI stops + // crashing on the wire-shape mismatch, but silently dropping it + // gave operators a fresh empty shell that lied about the + // snapshot. Surface an explicit 501 so the operator sees the gap + // (and the matching snapshot-then-restore workaround), in the + // envelope withCloneEnvelope picks for the caller: a CloneStarted + // object for python-linstor, an ApiCallRc array for everyone else. + // Note the LIVE-clone path below (Bug-020) takes its own internal + // snapshot — that is a different contract from honouring a + // caller-chosen point-in-time. + if req.SrcSnapName != "" { + writeSnapshotCloneNotImplemented(w, srcName, req.Name, req.SrcSnapName) + + return false + } + + return true +} + +// cloneRequestDropsNothing refuses the accepted-but-unhonoured fields. False +// means a refusal has been written and the caller must stop. +// +// Declaring a field so the decoder stops rejecting the body is only half the +// job. Accepting one and dropping it silently is the shape this endpoint +// already refuses for src_snap_name, and for the same reason: the caller is +// told the clone did what it asked, and it did something else. +// +// - external_name gives the definition an identity of its own upstream. +// Dropped, the clone comes back under a different name than requested. +// - volume_passphrases carries the LUKS keys for the cloned volumes. +// Dropped, the clone materialises with keys the caller does not hold. +// +// linstor-csi sends neither on this path, so refusing them costs nothing that +// works today and keeps the endpoint from lying if something starts to. +func cloneRequestDropsNothing(w http.ResponseWriter, srcName string, req *rdCloneRequest) bool { + if req.ExternalName != "" { + writeCloneRefused(w, http.StatusNotImplemented, srcName, req.Name, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + srcName + "': external_name is not implemented", + Cause: "blockstor names a cloned definition by `name`; honouring external_name would change the identity the caller asked for", + Correc: "omit external_name, or clone under the name you want", + }) + + return false + } + + if len(req.VolumePassphrases) > 0 { + writeCloneRefused(w, http.StatusNotImplemented, srcName, req.Name, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + srcName + "': volume_passphrases is not implemented", + Cause: "the clone would materialise with keys the caller does not hold, and report success", + Correc: "omit volume_passphrases; set the cluster passphrase with `linstor encryption create-passphrase` instead", + }) + + return false + } + + // The prop edits land on the target twice: folded in at create, where the + // marks are stamped over them, and again once the clone is done, after + // the marks, so an edit that rewrites or removes one would undo the clone + // under the caller: without the restore marker the satellite brings the + // volumes up blank and the retry no longer recognises the target as this + // clone. Refused rather than skipped, for the reason above: an edit the + // caller asked for and did not get is the silence this endpoint refuses. + // linstor-csi's own edit, deleting Aux/csi-provisioning-completed-by, + // touches none of them. + if key := store.ServerOwnedPropEdit(req.OverrideProps, req.DeleteProps, req.DeleteNamespaces); key != "" { + writeCloneRefused(w, http.StatusBadRequest, srcName, req.Name, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + srcName + "': the prop edits would change " + + key + ", which blockstor sets on the clone itself", + Cause: key + " records what the clone is; changing it in the same request undoes the clone", + Correc: "drop " + key + " from override_props, delete_props and delete_namespaces", + }) + + return false + } + + return true +} + +// cloneRequestShapeIsUsable validates the shape the caller picked for the +// clone: a layer stack that can be materialised at all, and a parent resource +// group that exists. The passphrase LUKS needs is checked after the replay of +// a finished clone, in clonedStackHasItsPassphrase, which writes no new LUKS +// volume and must not be refused over a passphrase that cannot be read. +func (s *Server) cloneRequestShapeIsUsable( + ctx context.Context, w http.ResponseWriter, srcName string, req *rdCloneRequest, +) bool { + // Validated the way rg-modify validates its stack, so an + // unmaterialisable layer chain is refused here rather than persisting + // onto the clone for a satellite to choke on. Stored in canonical case: + // the readers that ask whether a definition carries DRBD compare exactly, + // and a lowercase stack read as one without it. + err := validateLayerStack(req.LayerList) + if err == nil && len(req.LayerList) > 0 { + req.LayerList, err = validate.NormalizeLayerStack(req.LayerList) + } + + if err != nil { + writeCloneRefused(w, http.StatusBadRequest, srcName, req.Name, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + srcName + "': " + err.Error(), + }) + + return false + } + + return s.cloneResourceGroupExists(ctx, w, srcName, req) +} + +// clonedStackHasItsPassphrase refuses a clone whose stack carries LUKS while +// the cluster passphrase is missing: the stack it names, or the source's when +// it names none. It runs after the replay of a finished clone, which writes +// no new LUKS volume, so a passphrase that went missing or could not be read +// does not fail the retry of a clone that is already done. +func (s *Server) clonedStackHasItsPassphrase( + ctx context.Context, w http.ResponseWriter, src *apiv1.ResourceDefinition, req *rdCloneRequest, +) bool { + stack := req.LayerList + if len(stack) == 0 { + stack = store.EffectiveLayerStack(src.LayerStack) + } + + return s.clonePassphraseHolds(ctx, w, src.Name, req.Name, stack) +} + +// clonePassphraseHolds refuses a clone whose stack carries LUKS while the +// cluster passphrase is missing. False means a refusal has been written. +// +// Only the missing passphrase is the caller's to fix. A Secret or the +// controller props that could not be read is a failure of this side, and a +// 400 for it reads to linstor-csi as a permanent client error on a body it +// resends unchanged. +func (s *Server) clonePassphraseHolds( + ctx context.Context, w http.ResponseWriter, srcName, cloneName string, stack []string, +) bool { + luksErr := s.refuseLUKSWithoutPassphrase(ctx, stack) + if luksErr == nil { + return true + } + + status := http.StatusInternalServerError + if errors.Is(luksErr, ErrLUKSRequiresPassphrase) { + status = http.StatusBadRequest + } + + writeCloneRefused(w, status, srcName, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + srcName + "': " + luksErr.Error(), + }) + + return false +} + +// cloneResourceGroupExists applies the Bug 134 gate to the group a clone pins +// for itself. `resource_group` lands on the target on both clone paths — the +// shallow copy stamps it directly, the data-plane path hands it to +// materializeRestoredRD as a shape override — and neither went past the +// validator the RD-create path runs, so a typo produced a clone whose parent +// group does not exist. That RD lists fine and places badly: the placer's +// Controller→RG→RD prop walk drops the RG tier without a word, taking +// auto-place, auto-diskful, place_count and rebalance with it. +func (s *Server) cloneResourceGroupExists( + ctx context.Context, w http.ResponseWriter, srcName string, req *rdCloneRequest, +) bool { + found, err := s.lookupPinnedRG(ctx, req.ResourceGroup) + if err != nil { + writeCloneRefused(w, http.StatusInternalServerError, srcName, req.Name, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + srcName + "': " + scrubImplDetails(err.Error()), + }) + + return false + } + + if !found { + writeCloneRefused(w, http.StatusNotFound, srcName, req.Name, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + srcName + "': " + unknownRGMessage(req.ResourceGroup), + Correc: "create the resource group first, or omit resource_group to inherit the source's", + }) + + return false + } + + return true +} + // handleRDClone clones a ResourceDefinition under a new name. // // Two materialisation paths, switched on the source's @@ -124,39 +394,52 @@ func (s *Server) registerRDClone(mux *http.ServeMux) { // clone_bug_114_test.go (refusal contract for un-deployed sources) // and clone_use_zfs_clone_bug020_test.go (materialisation contract). func (s *Server) handleRDClone(w http.ResponseWriter, r *http.Request) { + w = withCloneEnvelope(w, r) srcName := r.PathValue("rd") var req rdCloneRequest - if !decodeJSON(w, r, &req) { + // Decoded without the shared answer: a malformed body, an unknown field or + // one over the size cap reaches python-linstor's clone decode too, and it + // reads `messages` off whatever comes back, so the array crashes it before + // the operator sees why their body was refused. + decodeErr := decodeJSONBody(r, &req) + if decodeErr != nil { + status, callRc := decodeErrorRc(decodeErr) + writeCloneRefused(w, status, srcName, req.Name, &callRc) + return } - if req.Name == "" { - writeError(w, http.StatusBadRequest, "name is required") + if !cloneRequestIsAnswerable(w, srcName, &req) { + return + } + if !s.rebindCloneTarget(r.Context(), w, srcName, &req) { return } - // Bug 239: clone-from-an-OPERATOR-NAMED-snapshot is not wired. - // The Bug 232 decoder accepts `src_snap_name` so the CLI stops - // crashing on the wire-shape mismatch, but silently dropping it - // gave operators a fresh empty shell that lied about the - // snapshot. Surface an explicit 501 + CloneStarted envelope so - // the operator sees the gap (and the matching snapshot-then- - // restore workaround). Note the LIVE-clone path below (Bug-020) - // takes its own internal snapshot — that is a different - // contract from honouring a caller-chosen point-in-time. - if req.SrcSnapName != "" { - writeSnapshotCloneNotImplemented(w, srcName, req.Name, req.SrcSnapName) + if !s.cloneRequestShapeIsUsable(r.Context(), w, srcName, &req) { + return + } + // A replay of a finished clone is answered before the source is read at + // all; see replayOfFinishedClone. The source may be gone, or hold no + // volumes, by the time a finished clone is replayed, and neither says + // anything about the clone. + replayed, halt := s.replayOfFinishedClone(r.Context(), w, srcName, &req) + if replayed || halt { return } src, err := s.Store.ResourceDefinitions().Get(r.Context(), srcName) if err != nil { - writeStoreError(w, err) + writeCloneStoreError(w, srcName, req.Name, err) + + return + } + if !s.clonedStackHasItsPassphrase(r.Context(), w, &src, &req) { return } @@ -165,7 +448,10 @@ func (s *Server) handleRDClone(w http.ResponseWriter, r *http.Request) { // contract Group D pins. srcVDs, err := s.Store.VolumeDefinitions().List(r.Context(), srcName) if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) + writeCloneRefused(w, http.StatusInternalServerError, srcName, req.Name, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: scrubImplDetails(err.Error()), + }) return } @@ -179,6 +465,60 @@ func (s *Server) handleRDClone(w http.ResponseWriter, r *http.Request) { s.cloneEmptyRDShell(w, r, &src, &req) } +// cloneTargetNameIsUsable holds the clone door to the identifier rules every +// other door that creates a definition runs: `rd create`, `spawn` and the +// restore all validate the name they are handed, and a clone that skipped them +// could create a definition `rd create` answers 400 for, which +// tests/e2e/rd-name-validation-bulk.sh treats as a contract. +// +// The internal snapshot's name is not checked here: only the data path takes +// one, and a volume-less clone under a name `rd create` accepts has to stay +// possible. cloneSnapshotNameIsUsable checks it on that path. +func cloneTargetNameIsUsable(w http.ResponseWriter, srcName string, req *rdCloneRequest) bool { + if req.Name == "" { + writeCloneRefused(w, http.StatusBadRequest, srcName, req.Name, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "name is required", + }) + + return false + } + + nameErr := validateLinstorName("resource definition", req.Name) + if nameErr != nil { + writeCloneRefused(w, http.StatusBadRequest, srcName, req.Name, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + srcName + "': " + nameErr.Error(), + }) + + return false + } + + return true +} + +// cloneSnapshotNameIsUsable refuses a target whose internal snapshot name +// would pass the identifier ceiling. The name is derived by prefixing the +// target, so a target just inside the ceiling would take the clone through a +// snapshot create the store refuses, and it is checked before anything is +// written. +func cloneSnapshotNameIsUsable(w http.ResponseWriter, srcName, cloneName string) bool { + nameErr := validateLinstorName("snapshot", cloneSnapshotName(cloneName)) + if nameErr == nil { + return true + } + + writeCloneRefused(w, http.StatusBadRequest, srcName, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + srcName + "' into '" + cloneName + + "': the internal snapshot this clone takes would be named '" + + cloneSnapshotName(cloneName) + "': " + nameErr.Error(), + Correc: "pick a shorter target name", + }) + + return false +} + // cloneWithData materialises a clone of a VD-bearing source RD by // routing through the snapshot-restore machinery (Bug-020): // @@ -204,44 +544,298 @@ func (s *Server) handleRDClone(w http.ResponseWriter, r *http.Request) { func (s *Server) cloneWithData(w http.ResponseWriter, r *http.Request, src *apiv1.ResourceDefinition, req *rdCloneRequest) { ctx := r.Context() - if s.cloneTargetPreexists(ctx, w, src.Name, req.Name) { + if !cloneSnapshotNameIsUsable(w, src.Name, req.Name) { return } - // Mirror the snapshot-create Bug 180 gate: a source RD mid-tear- - // down would reap the internal snapshot + clone marker from - // under the satellite's restore. - if rdHasDeleteFlag(ctx, s, src.Name) { - writeCloneRefused(w, http.StatusConflict, src.Name, req.Name, &apiv1.APICallRc{ - RetCode: apiCallRcError, - Message: "clone of resource definition '" + src.Name + "' refused: the source is being deleted", - Cause: "the source RD carries the DELETE flag; its backing data is being torn down", - Correc: "clone before deleting the source, or restore from a snapshot taken earlier", - }) + if !cloneLayerStackIsHonourable(w, src, req) { + return + } + + resume, stop := s.cloneTargetState(ctx, w, src, req) + if stop { + return + } + if !s.cloneSourceIsNotBeingDeleted(ctx, w, src, req.Name) { return } - snap, ok := s.ensureCloneSnapshot(w, r, src, req.Name) + if resume && !s.cloneResumeKeepsItsPointInTime(ctx, w, src, req.Name) { + return + } + + snap, reusedSnapshot, ok := s.ensureCloneSnapshot(w, r, src, req.Name) if !ok { return } + // The only question a leftover still raises: does a REUSED snapshot still + // describe the source? It belongs to a clone about to be materialised and + // never to one already made, which is why the finished question is asked + // at the top rather than here. + if reusedSnapshot && !s.cloneSnapshotIsCurrent(ctx, w, src, snap, req.Name) { + return + } + + s.materializeClone(ctx, w, src, req, snap, resume) +} + +// cloneResumeKeepsItsPointInTime refuses to resume an unfinished clone whose +// internal snapshot is gone once the leftover holds anything restored from it. +// False means a refusal has been written. +// +// ensureCloneSnapshot takes the snapshot again when it is missing, of the +// source as it is now. A leftover that already has a volume with a replica was +// restored from the earlier point-in-time, and finishing it from the later one +// stitches two moments of the source into one clone: a source whose data moved +// on without its volume sizes changing passes every shape check and comes back +// as a clone nobody asked for. A leftover with no volume, or with no replica, +// holds no data of the earlier moment, and retaking the snapshot makes it the +// fresh clone it would have been, as long as its volumes fit the source as it +// is now (leftoverFitsARetake). +// +// The refusal names its way out, so linstor-csi's retry, which polls the +// status and POSTs again on a 404, ends at an operator rather than looping. +func (s *Server) cloneResumeKeepsItsPointInTime( + ctx context.Context, w http.ResponseWriter, src *apiv1.ResourceDefinition, cloneName string, +) bool { + // Every read here is the API server's: the answer refuses a clone the + // operator then deletes by hand, and a cache that has not seen the + // snapshot, or still lists a replica being deleted, would make that call + // for a clone that is fine. + err := cloneSnapshotIsPresent(ctx, s.Store, src.Name, cloneSnapshotName(cloneName)) + if err == nil { + return true + } + + if !errors.Is(err, store.ErrNotFound) { + writeCloneRefused(w, http.StatusInternalServerError, src.Name, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + src.Name + "': the snapshot behind '" + + cloneName + "' could not be read: " + scrubImplDetails(err.Error()), + Correc: "retry the clone", + }) + + return false + } + + vds, err := store.LiveVolumes(ctx, s.Store, cloneName) + if err != nil { + writeCloneStoreError(w, src.Name, cloneName, err) + + return false + } + + count, err := countReplicas(ctx, s.Store, cloneName) + if err != nil { + writeCloneStoreError(w, src.Name, cloneName, err) + + return false + } + + // Data from the earlier moment exists only where a volume has a replica + // holding it: a volume definition alone is a size, a replica with no + // volume holds nothing, one stamped for deletion is going with its data, + // and a diskless one never had any. + // Either half on its own is finished from the new snapshot like the fresh + // clone it still is, once its volumes are known to fit that snapshot. + if len(vds) == 0 { + return true + } + + if count.Holding == 0 { + return s.leftoverFitsARetake(ctx, w, src, cloneName, vds) + } + + writeCloneRefused(w, http.StatusConflict, src.Name, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError | apiCallRcFailExistsRscDfn, + Message: "clone target '" + cloneName + "' is an unfinished clone whose snapshot '" + + cloneSnapshotName(cloneName) + "' is gone", + Cause: "the leftover already holds data restored from that snapshot, and finishing it " + + "from a new snapshot of the source as it is now would join two different moments " + + "of the source into one clone", + Correc: "delete '" + cloneName + "' and clone again", + }) + + return false +} + +// cloneSnapshotIsPresent reads one snapshot from the API server, answering +// store.ErrNotFound when it is not there. +func cloneSnapshotIsPresent(ctx context.Context, st store.Store, srcName, snapName string) error { + _, err := liveCloneSnapshot(ctx, st, srcName, snapName) + + return err +} + +// cloneSnapshotIsReusable judges the internal snapshot an earlier attempt left. +// False means a refusal has been written. +func (s *Server) cloneSnapshotIsReusable( + ctx context.Context, w http.ResponseWriter, src *apiv1.ResourceDefinition, existing *apiv1.Snapshot, cloneName string, +) bool { + // Interrupted-clone retry: the snapshot landed on a previous + // attempt; reuse it so the restore sees the same point-in-time. + // + // Whether it still describes the SOURCE is a question for the + // caller, and only when the clone is unfinished — a finished one is + // a copy of the point-in-time and owes the source nothing. + // + // What is checked here is what the create branch checks and this one + // used to skip: a snapshot recording no nodes places no replicas, so + // the clone would answer 201 over an empty shell, which is the Bug + // 114 shape the create branch refuses. + if len(existing.Nodes) == 0 { + writeCloneRefused(w, http.StatusConflict, src.Name, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + src.Name + "' refused: the leftover snapshot '" + + existing.Name + "' records no nodes", + Cause: "a snapshot with no nodes places no replicas, so the clone would " + + "report success over an empty shell", + Correc: "delete the snapshot '" + existing.Name + "' so the clone retakes it", + }) + + return false + } + + return s.cloneSnapshotNodesAreUsable(ctx, w, src, existing, cloneName) +} + +// leftoverFitsARetake refuses, before a new snapshot is taken, a leftover +// with no replica whose volumes the source as it is now would not fit: one +// smaller than the source's volume of that number, or one the source no +// longer has. Over a smaller volume the retake snapshotted the source and then +// failed hydrating with a bare already-exists, leaving a snapshot of a +// point-in-time the leftover was never restored from; over a volume the +// source no longer has it answered complete, keeping beside the retake a +// volume nothing restored. False means a refusal has been written. +func (s *Server) leftoverFitsARetake( + ctx context.Context, w http.ResponseWriter, src *apiv1.ResourceDefinition, cloneName string, + vds []apiv1.VolumeDefinition, +) bool { + srcVDs, err := store.LiveVolumes(ctx, s.Store, src.Name) + if err != nil { + writeCloneStoreError(w, src.Name, cloneName, err) + + return false + } + + sizes := make(map[int32]int64, len(srcVDs)) + for i := range srcVDs { + sizes[srcVDs[i].VolumeNumber] = srcVDs[i].SizeKib + } + + for i := range vds { + size, ok := sizes[vds[i].VolumeNumber] + if ok && vds[i].SizeKib >= size { + continue + } + + writeCloneRefused(w, http.StatusConflict, src.Name, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError | apiCallRcFailExistsRscDfn, + Message: "clone target '" + cloneName + "' is an unfinished clone whose snapshot '" + + cloneSnapshotName(cloneName) + "' is gone, and its volumes do not fit the source as it is now", + Cause: "finishing it would snapshot the source again and hydrate around a volume of " + + "another size, or one the source no longer has", + Correc: "delete '" + cloneName + "' and clone again", + }) + + return false + } + + return true +} + +// readCloneSnapshot reads the clone's internal snapshot past the cache, as +// cloneResumeKeepsItsPointInTime reads it: a cache that had not seen the +// snapshot's delete yet took the reuse branch that read had just ruled out, +// and stamped replicas restoring from a snapshot that no longer exists. It +// returns (snapshot, found, ok); !ok means a refusal has been written. +func readCloneSnapshot( + ctx context.Context, w http.ResponseWriter, st store.Store, srcName, cloneName string, +) (apiv1.Snapshot, bool, bool) { + snapName := cloneSnapshotName(cloneName) + + snap, err := liveCloneSnapshot(ctx, st, srcName, snapName) + if errors.Is(err, store.ErrNotFound) { + return apiv1.Snapshot{}, false, true + } + + if err != nil { + writeCloneRefused(w, http.StatusInternalServerError, srcName, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + srcName + "' failed: read the internal snapshot '" + + snapName + "': " + scrubImplDetails(err.Error()), + Correc: "retry the clone", + }) + + return apiv1.Snapshot{}, false, false + } + + return snap, true, true +} + +// liveCloneSnapshot reads the clone's internal snapshot from the API server. +func liveCloneSnapshot(ctx context.Context, st store.Store, srcName, snapName string) (apiv1.Snapshot, error) { + snaps, err := st.Snapshots().ListByDefinitionUncached(ctx, srcName) + if err != nil { + return apiv1.Snapshot{}, err //nolint:wrapcheck // surfaced via writeCloneRefused with the source named + } + + for i := range snaps { + if strings.EqualFold(snaps[i].Name, snapName) { + return snaps[i], nil + } + } + + return apiv1.Snapshot{}, errors.Wrapf(store.ErrNotFound, "snapshot %q of %q", snapName, srcName) +} + +// materializeClone restores the target from the clone's snapshot, guards the +// group it was written with, applies the prop edits and answers. resume says +// whether this request is finishing a leftover, which only the wording sees: +// whether the definition is this request's to roll back is what +// materializeRestoredRD found, not what the gate expected. +func (s *Server) materializeClone( + ctx context.Context, w http.ResponseWriter, src *apiv1.ResourceDefinition, req *rdCloneRequest, + snap *apiv1.Snapshot, resume bool, +) { restoreReq := &snapshotRestoreRequest{ToResource: req.Name} // Clone path: eagerPlace=true. `rd clone` is a one-shot CSI // operation with no follow-up autoplace, so the clone replicas must // materialise on the snapshot-holding nodes in the source pool here // (same backend by construction — Bug 038). - made, err := s.materializeRestoredRD(ctx, src.Name, restoreReq, snap, true) + made, err := s.materializeRestoredRD(ctx, src.Name, restoreReq, snap, true, &rdShapeOverrides{ + LayerStack: req.LayerList, + ResourceGroupName: req.ResourceGroup, + OverrideProps: req.OverrideProps, + DeleteProps: req.DeleteProps, + DeleteNamespaces: req.DeleteNamespaces, + }) if err != nil { + // The same split the restore door draws: an answer about what was + // already there keeps its typed status (and band, where it has one; + // see materialiseRefusalKind), and only this request's own partial + // work goes through the rollback. + if kind, refused := materialiseRefusalKind(err); refused { + status, rc := storeErrorRc(err, kind) + writeCloneRefused(w, status, src.Name, req.Name, &rc) + + return + } + writeCloneRefused(w, http.StatusInternalServerError, src.Name, req.Name, s.failedMaterialiseRefusal(ctx, "clone of resource definition '"+src.Name+"' failed: "+err.Error(), - "clone", req.Name, made.Placed, err)) + "clone", req.Name, made.Placed, rollbackUnlessPlaced, nil, err)) return } + // Named from here on as it is stored: a definition adopted through a + // collided create may carry another spelling than the request's. + req.Name = made.Name + uncheckedRG, ok := s.cloneParentRGSurvived(ctx, w, src, req.Name, made) if !ok { return @@ -252,13 +846,25 @@ func (s *Server) cloneWithData(w http.ResponseWriter, r *http.Request, src *apiv writeCloneRefused(w, http.StatusInternalServerError, src.Name, req.Name, &apiv1.APICallRc{ RetCode: apiCallRcError, Message: "clone of resource definition '" + src.Name + "' created, but applying " + - "override_props/delete_props failed: " + err.Error(), + "override_props/delete_props failed: " + scrubImplDetails(err.Error()), }) return } - writeCloneStarted(w, src.Name, req.Name, "resource definition cloned: "+req.Name, uncheckedRG) + writeCloneStarted(w, src.Name, req.Name, cloneDoneMessage(resume, req.Name), uncheckedRG) +} + +// cloneDoneMessage names whether the clone completed a leftover, so a retry +// reads as one in the caller's own output instead of looking like a first +// run. Mirrors restoreDoneMessage, which draws the same distinction on the +// endpoint this one shares its data plane with. +func cloneDoneMessage(resumed bool, cloneName string) string { + if resumed { + return "resource definition clone completed on retry: " + cloneName + } + + return "resource definition cloned: " + cloneName } // correcRecreateGroupThenClone is the one wording both rollback doors on this @@ -322,9 +928,9 @@ func (s *Server) cloneParentRGSurvived( return nil, false } - rollbackErr := s.rollBackCompensating(ctx, cloneName, made.Placed) + rollbackErr := s.rollBackCompensating(ctx, cloneName, made.Placed, rollbackEvenIfFinished) if rollbackErr != nil { - cause, correc := rollbackFailureAdvice(rollbackErr, cloneName) + cause, correc := rollbackFailureAdviceOverDeletedGroup(rollbackErr, cloneName, stampedRG) writeCloneRefused(w, http.StatusInternalServerError, src.Name, cloneName, &apiv1.APICallRc{ RetCode: apiCallRcError, @@ -356,7 +962,7 @@ func uncheckedCloneGroupWarning(cloneName, rgName string, err error) *apiv1.APIC return &apiv1.APICallRc{ RetCode: maskWarn, Message: "resource group '" + rgName + "' could not be re-checked after the " + - "clone: " + err.Error(), + "clone: " + scrubImplDetails(err.Error()), Cause: "the clone itself succeeded; only the safety net over it could not be " + "inspected, so a group deleted during the clone would not have been caught", Correc: "confirm resource group '" + rgName + "' still exists", @@ -367,145 +973,198 @@ func uncheckedCloneGroupWarning(cloneName, rgName string, err error) *apiv1.APIC } } -// cloneSnapshotName derives the internal snapshot name backing a -// data-plane clone. Deterministic (`clone-`) so an -// interrupted clone retried by linstor-csi reuses the same snapshot -// instead of accreting one per attempt. The snapshot is visible in -// `linstor s l` like any other — it must outlive the clone because -// `zfs clone` targets stay dependent on their origin snapshot. -func cloneSnapshotName(cloneName string) string { - return "clone-" + cloneName -} - -// cloneTargetPreexists handles the clone-target-already-exists edge -// up front (true = response already written): +// cloneLayerStackIsHonourable refuses a clone whose requested layer stack is +// not the source's. False means a refusal has been written and the caller must +// stop. // -// - target carrying OUR restore marker for this exact source + -// internal snapshot → idempotent retry of a clone that already -// materialised (linstor-csi replays CreateVolume until it sees -// success); answer 201 + the same CloneStarted envelope. -// - any other pre-existing RD under that name → 409 refusal in -// CloneStarted shape (a bare store AlreadyExists envelope would -// crash python-linstor's clone decode). -func (s *Server) cloneTargetPreexists(ctx context.Context, w http.ResponseWriter, srcName, cloneName string) bool { - existing, err := s.Store.ResourceDefinitions().Get(ctx, cloneName) - if err != nil { - // NotFound (or any read blip) → proceed with the create; - // a real store outage surfaces on the next write anyway. - return false +// The clone data plane restores the source's bytes onto the target and brings +// the layer stack up over them, in that order. Every layer's bring-up writes +// to the device it is given: +// +// - LUKS: luks.Format treats a device carrying no LUKS header as one to +// format, so adding LUKS over a plaintext source formats away the data +// that was just restored; +// - DRBD: create-md runs with --force (pkg/drbd/drbdadm.go) over +// `meta-disk internal` (pkg/drbd/conffile.go), stamping metadata across +// the tail of the same bytes. The only gate before it is HasMD, which +// looks for DRBD metadata and never for a filesystem signature. +// +// Both directions are refused, and the rule is the whole set rather than a +// list of the layers known to write today: a clone is a copy, so a target of a +// different shape does not hold the source's data whichever layer differs, and +// a layer added to LINSTOR later inherits the refusal instead of a gap. +// Ordering and spelling are not a difference — the stack's order follows from +// the kinds in it, and LINSTOR folds name case. +// +// Only the data-bearing path calls this. A clone of a VD-less source carries +// no bytes to lose, and choosing a different stack for the shell is what +// accepting layer_list is for. +func cloneLayerStackIsHonourable(w http.ResponseWriter, src *apiv1.ResourceDefinition, req *rdCloneRequest) bool { + if len(req.LayerList) == 0 { + return true } - if existing.Props["BlockstorRestoreFromSnapshot"] == srcName+":"+cloneSnapshotName(cloneName) { - if !s.cloneLeftoverIsUsable(ctx, w, srcName, cloneName, &existing) { - return true - } - - writeJSON(w, http.StatusCreated, cloneStartedResponse{ - Location: "/v1/resource-definitions/" + srcName + "/clone/" + cloneName, - SourceName: srcName, - CloneName: cloneName, - Messages: &[]apiv1.APICallRc{{ - RetCode: maskInfo, - Message: "resource definition already cloned: " + cloneName, - }}, - }) - + // An RD stored without an explicit stack is not a definition with no + // layers — it is one that never said, and every other reader resolves + // that to apiv1.DefaultLayerStack (stampRDLayerDataFromStack does it on + // the read path). Resolving it here too is what keeps the gate off the + // CSI hot path: linstor-csi sends [DRBD, STORAGE] on every clone, and + // against a bare source an unresolved empty stack would read as "adds + // DRBD, STORAGE" and refuse the clone this endpoint was just unbroken + // for. + have := store.EffectiveLayerStack(src.LayerStack) + + added, dropped := layerSetDifference(have, req.LayerList) + if len(added) == 0 && len(dropped) == 0 { return true } - writeCloneRefused(w, http.StatusConflict, srcName, cloneName, &apiv1.APICallRc{ + writeCloneRefused(w, http.StatusBadRequest, src.Name, req.Name, &apiv1.APICallRc{ RetCode: apiCallRcError, - Message: "clone target '" + cloneName + "' already exists and is not a clone of '" + srcName + "'", - Correc: "pick a different clone name, or delete the existing resource definition first", + Message: "clone of resource definition '" + src.Name + "': layer_list " + + describeLayerDifference(added, dropped), + Cause: "the clone restores the source's data first and brings the layer stack up " + + "over it, so a layer added here writes its own metadata across the bytes it " + + "just restored, and one dropped leaves the target reading data the missing " + + "layer wrote", + Correc: "clone with the source's own layer stack; a definition of a different " + + "shape has to be created and copied into", }) - return true + return false } -// cloneLeftover is what a replay found under a marker-bearing definition. -type cloneLeftover int +// layerSetDifference reports which layers the requested stack adds to the +// source's and which it drops. Case-insensitive, because LINSTOR names fold +// and the two stacks reach here from different writers. +func layerSetDifference(have, want []string) ([]string, []string) { + var added, dropped []string -const ( - // cloneLeftoverWhole holds volumes and at least one replica that is not - // being deleted: the least a replay may answer 201 over. - cloneLeftoverWhole cloneLeftover = iota - // cloneLeftoverTearingDown still lists replicas, but every one of them is - // already accepted for deletion. - cloneLeftoverTearingDown - // cloneLeftoverEmpty has neither volumes nor replicas. - cloneLeftoverEmpty - // cloneLeftoverNoVolumes has replicas and no volumes. - cloneLeftoverNoVolumes - // cloneLeftoverNoReplicas has volumes and no replica at all. - cloneLeftoverNoReplicas -) - -// assessCloneLeftover reads a marker-bearing definition until it is whole or -// the cache-retry budget the parent-group read carries is spent. -// -// Both reads are informer-cache served, and the informer that matched the -// marker is not the one answering the replica listing, so a definition seen -// with its replicas not yet seen is the ordinary skew that budget exists for. -// Deciding on the first read told a complete clone to delete itself. A NotFound -// counts as "not seen yet" for the same reason; any other read error is -// returned at once, and the caller refuses on it. -func (s *Server) assessCloneLeftover(ctx context.Context, cloneName string) (cloneLeftover, error) { - var state cloneLeftover - - for attempt := range cacheRetryAttempts { - var err error - - state, err = s.readCloneLeftover(ctx, cloneName) - if err != nil || state == cloneLeftoverWhole || attempt == cacheRetryAttempts-1 { - return state, err + for _, layer := range want { + if !apiv1.LayerInStack(have, layer) { + added = append(added, strings.ToUpper(layer)) } + } - select { - case <-ctx.Done(): - return state, errors.Wrapf(ctx.Err(), "re-read the leftover %q", cloneName) - case <-time.After(cacheRetryDelay): + for _, layer := range have { + if !apiv1.LayerInStack(want, layer) { + dropped = append(dropped, strings.ToUpper(layer)) } } - return state, nil + return added, dropped } -// readCloneLeftover is one read of a marker-bearing definition's volumes and -// replicas. A replica stamped for deletion is not counted as holding the -// clone: its satellite finalizer may keep it listed for as long as the owning -// node is down, and a replay answered 201 over it binds a volume to a clone -// that is going away. -func (s *Server) readCloneLeftover(ctx context.Context, cloneName string) (cloneLeftover, error) { - vds, err := s.Store.VolumeDefinitions().List(ctx, cloneName) - if err != nil && !errors.Is(err, store.ErrNotFound) { - return cloneLeftoverEmpty, errors.Wrapf(err, "list the volumes of %q", cloneName) +// describeLayerDifference names what the caller asked to change, so the +// refusal says which layer rather than only that the stacks differ. +func describeLayerDifference(added, dropped []string) string { + switch { + case len(added) > 0 && len(dropped) > 0: + return "adds " + strings.Join(added, ", ") + " and drops " + + strings.Join(dropped, ", ") + " relative to the source" + case len(added) > 0: + return "adds " + strings.Join(added, ", ") + " to the source's stack" + default: + return "drops " + strings.Join(dropped, ", ") + " from the source's stack" } +} - replicas, err := s.Store.Resources().ListByDefinition(ctx, cloneName) - if err != nil && !errors.Is(err, store.ErrNotFound) { - return cloneLeftoverEmpty, errors.Wrapf(err, "list the replicas of %q", cloneName) +// cloneSnapshotName derives the internal snapshot name backing a +// data-plane clone. Deterministic (`clone-`) so an +// interrupted clone retried by linstor-csi reuses the same snapshot +// instead of accreting one per attempt. The snapshot is visible in +// `linstor s l` like any other — it must outlive the clone because +// `zfs clone` targets stay dependent on their origin snapshot. +func cloneSnapshotName(cloneName string) string { + return "clone-" + cloneName +} + +// writeCloneTargetUnreadable refuses a clone whose target could not be read. +// Read as absent, the create that follows collides with what is there and is +// adopted past every judgement of a leftover (finished, torn down, parented to +// a group that is gone, taken from a later point-in-time), so a replay of a +// finished clone re-placed it; the retry this answer asks for gets the +// judgement. +func writeCloneTargetUnreadable(w http.ResponseWriter, srcName, cloneName string, err error) { + writeCloneRefused(w, http.StatusInternalServerError, srcName, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone target '" + cloneName + "' could not be read: " + scrubImplDetails(err.Error()), + Correc: "retry the clone", + }) +} + +// cloneTargetState decides what a definition already under the clone's target +// name means. It returns (resume, stop): stop when an answer has already been +// written, resume when the caller should re-run the clone over the leftover. +// +// The restore marker is what makes a leftover recognisable, and it is NOT +// evidence that the clone finished. materializeRestoredRD stamps it with the +// definition, then hydrates the volumes and places the replicas, so a failure +// in either leaves the marker sitting on an empty shell. Answering 201 on the +// marker alone — which this did — turns the retry linstor-csi issues after a +// partial failure into a silent incomplete: CSI sees the volume as ready and +// nothing ever finishes it. Re-running is safe because every clone step +// tolerates an object a previous attempt already created, so it completes what +// is missing and leaves what is there. Same split restoreTargetState draws on +// the restore path, for the same reason. +// +// A leftover mid-tear-down is refused rather than resumed: the deletion is +// reaping the very objects completing the clone would be writing. +// +// Anything else under that name is a genuine collision and stays a refusal, in +// CloneStarted shape — a bare store AlreadyExists envelope would crash +// python-linstor's clone decode. +func (s *Server) cloneTargetState( + ctx context.Context, w http.ResponseWriter, src *apiv1.ResourceDefinition, req *rdCloneRequest, +) (bool, bool) { + srcName, cloneName := src.Name, req.Name + + existing, err := s.Store.ResourceDefinitions().Get(ctx, cloneName) + if errors.Is(err, store.ErrNotFound) { + return false, false } - live := 0 + if err != nil { + writeCloneTargetUnreadable(w, srcName, cloneName, err) - for i := range replicas { - if !replicaAcceptedForDeletion(&replicas[i]) { - live++ - } + return false, true } - switch { - case len(vds) > 0 && live > 0: - return cloneLeftoverWhole, nil - case len(replicas) > 0 && live == 0: - return cloneLeftoverTearingDown, nil - case len(vds) == 0 && len(replicas) == 0: - return cloneLeftoverEmpty, nil - case len(vds) == 0: - return cloneLeftoverNoVolumes, nil - default: - return cloneLeftoverNoReplicas, nil + if !restoreMarkerMatches(existing.Props, srcName, cloneSnapshotName(cloneName)) { + writeCloneRefused(w, http.StatusConflict, srcName, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone target '" + cloneName + "' already exists and is not a clone of '" + srcName + "'", + Correc: "pick a different clone name, or delete the existing resource definition first", + }) + + return false, true + } + + if slices.Contains(existing.Flags, rdFlagDelete) { + writeCloneRefused(w, http.StatusConflict, srcName, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone target '" + cloneName + "' is being deleted", + Cause: "the leftover from an earlier attempt at this clone carries the DELETE " + + "flag; completing it would race the tear-down reaping what it writes", + Correc: "wait for the delete to finish, then re-issue the clone", + }) + + return false, true } + + // The requested shape is not compared here. replayOfFinishedClone runs + // first on the same leftover and the same fields and refuses a shape the + // leftover was not started with, finished or not, so a second comparison + // could only repeat its answer. + // + // The group and the abandoned-rollback mark are: resuming over a leftover + // whose rollback gave up, or one parented to a group that is gone, would + // finish exactly the definition those guards exist to stop. + if !s.cloneLeftoverIsUsable(ctx, w, srcName, &existing) { + return false, true + } + + return true, false } // cloneShellParentRGSurvived is the vol-less half of the same guard. @@ -555,12 +1214,12 @@ func (s *Server) cloneShellParentRGSurvived( // shell exactly where it was, parented to a group that is gone. The data // path refuses to make that claim over a failed compensation, and the two // halves of one guard should not answer the same question differently. - err = s.rollBackCompensating(ctx, cloneName, nil) + err = s.rollBackCompensating(ctx, cloneName, nil, rollbackEvenIfFinished) if err != nil { // The advice the shared rollback's other doors give, for the step // that failed: a snapshot on the shell makes "delete it by hand" a // dead end, since `rd d` refuses a definition that has snapshots. - cause, correc := rollbackFailureAdvice(err, cloneName) + cause, correc := rollbackFailureAdviceOverDeletedGroup(err, cloneName, stampedRG) writeCloneRefused(w, http.StatusInternalServerError, srcName, cloneName, &apiv1.APICallRc{ RetCode: apiCallRcError, @@ -599,42 +1258,41 @@ func (s *Server) cloneShellParentRGSurvived( // The advice in the 500 never reaches a human, because the machine turns the // failure into a success first. // -// So a replay is answered 201 only over a leftover that is whole and whose -// parent group still resolves. An inconclusive read of either refuses: unlike -// the post-write check, which guards a clone this request has just made, this -// gate would otherwise report a clone nobody verified, and the CSI retry makes -// a refusal cheap. +// So a leftover is answered for, replayed as finished or resumed, only when +// its parent group still resolves and no rollback gave up over it. An +// inconclusive read of either refuses: unlike the post-write check, which +// guards a clone this request has just made, this gate would otherwise report +// a clone nobody verified, and the CSI retry makes a refusal cheap. Whether +// the leftover is whole is assessMarkedClone's question, asked by the caller: +// a finished one is replayed, an unfinished one resumed, and one whose every +// replica is being torn down refused. +// +// The clone is named by the leftover's stored spelling, not the request's: +// LINSTOR folds name case, and the uncached read the mark gate takes is keyed +// by what was stored. func (s *Server) cloneLeftoverIsUsable( - ctx context.Context, w http.ResponseWriter, srcName, cloneName string, existing *apiv1.ResourceDefinition, + ctx context.Context, w http.ResponseWriter, srcName string, existing *apiv1.ResourceDefinition, ) bool { - stampedRG := existing.ResourceGroupName - // The group resolving is not enough on its own. A failed rollback may - // have reaped every replica and still kept the definition, and both of - // this guard's corrections tell the operator to re-create the group — - // so the moment they do, a group-only gate answers 201 for a clone that - // exists on no node. The leftover has to be whole. - state, err := s.assessCloneLeftover(ctx, cloneName) - if err != nil { - writeCloneRefused(w, http.StatusInternalServerError, srcName, cloneName, &apiv1.APICallRc{ - RetCode: apiCallRcError, - Message: "clone target '" + cloneName + "' exists, but reading it back failed: " + err.Error(), - Cause: "without its volumes and replicas the replay cannot tell a finished clone " + - "from an unfinished one, and answering 201 over the second binds a volume to " + - "a clone that may exist on no node", - Correc: "retry the clone", - }) - - return false + status, refusal := s.cloneLeftoverRefusal(ctx, existing) + if refusal == nil { + return true } - if state != cloneLeftoverWhole { - writeCloneRefused(w, http.StatusConflict, srcName, cloneName, cloneLeftoverRefusal(cloneName, state)) + writeCloneRefused(w, status, srcName, existing.Name, refusal) - return false - } + return false +} + +// cloneLeftoverRefusal is cloneLeftoverIsUsable's judgement without the +// answer, so the status poll asks exactly what the POST asks. Nil means the +// leftover may be answered for. +func (s *Server) cloneLeftoverRefusal( + ctx context.Context, existing *apiv1.ResourceDefinition, +) (int, *apiv1.APICallRc) { + stampedRG, cloneName := existing.ResourceGroupName, existing.Name if stampedRG == "" { - return !s.cloneRollbackWasAbandoned(ctx, w, srcName, cloneName) + return s.abandonedRollbackRefusal(ctx, "clone", cloneName) } // An unreadable group is refused rather than waved through. Refusing @@ -647,23 +1305,21 @@ func (s *Server) cloneLeftoverIsUsable( // not true, where the honest answer is to try again. survived, err := s.parentRGSurvived(ctx, stampedRG) if err != nil { - writeCloneRefused(w, http.StatusInternalServerError, srcName, cloneName, &apiv1.APICallRc{ + return http.StatusInternalServerError, &apiv1.APICallRc{ RetCode: apiCallRcError, Message: "clone target '" + cloneName + "' exists, but its parent resource group '" + - stampedRG + "' could not be read: " + err.Error(), + stampedRG + "' could not be read: " + scrubImplDetails(err.Error()), Cause: "the replay only answers for a clone whose parent group resolves, and this " + "read failed rather than saying the group is gone", Correc: "retry the clone", - }) - - return false + } } if survived { - return !s.cloneRollbackWasAbandoned(ctx, w, srcName, cloneName) + return s.abandonedRollbackRefusal(ctx, "clone", cloneName) } - writeCloneRefused(w, http.StatusConflict, srcName, cloneName, &apiv1.APICallRc{ + return http.StatusConflict, &apiv1.APICallRc{ RetCode: apiCallRcError, Message: "clone target '" + cloneName + "' exists but is parented to resource group '" + stampedRG + "', which no longer exists", @@ -673,106 +1329,106 @@ func (s *Server) cloneLeftoverIsUsable( "clone that is not usable", Correc: "delete '" + cloneName + "' by hand, re-create resource group '" + stampedRG + "', then clone again", - }) + } +} - return false +// clearRollbackMarkCommand is the command that takes a rollback's mark off a +// definition the rollback left whole; see store.ServerOwnedPropEditByOperator. +// python-linstor has no delete-property: a set-property with no value is its +// delete, and the blockstor CLI reads it the same way. +func clearRollbackMarkCommand(rdName string) string { + return "run `linstor resource-definition set-property " + rdName + " " + rollbackAbandonedKey + + "` with no value" } -// cloneRollbackWasAbandoned refuses, as the last word before a replay would -// answer 201, a leftover whose rollback gave up. True means the refusal has -// been written. +// abandonedRollbackRefusal is the abandoned-rollback gate both resuming doors +// share: the clone and the restore write the mark through the same failed +// materialisation, so both have to read it back before resuming, or one door +// refuses the leftover its twin finishes. Nil means nothing to refuse. // // It comes last on purpose. Every earlier refusal is more precise about the // same leftover (still being torn down, parented to a group that is gone), and -// this one only has to catch what they all let through: a leftover that looks -// whole because the placement the rollback stopped in the middle of left a -// live replica and the volumes, when the clone intended more replicas than -// that. The rollback is the one party that knows; see rollbackAbandonedKey. +// this one only has to catch what they all let through: a leftover a cascade +// was cut short over, see rollbackAbandonedKey. // // The definition is read again here, from the API server. The props the gate -// started from are one cache-served read taken before the two gates ahead of -// this one waited out their own cache lag, and the mark is written through the -// API server by a rollback that may have given up moments before this replay +// started from are one cache-served read taken before the gates ahead of this +// one waited out their own cache lag, and the mark is written through the API +// server by a rollback that may have given up moments before the replay // arrived, which is exactly when the cache has not caught up with it. A read -// that fails refuses: the replay cannot vouch for a clone it could not check. -func (s *Server) cloneRollbackWasAbandoned( - ctx context.Context, w http.ResponseWriter, srcName, cloneName string, -) bool { - existing, err := s.Store.ResourceDefinitions().GetUncached(ctx, cloneName) +// that fails refuses: the replay cannot vouch for a definition it could not +// check. +func (s *Server) abandonedRollbackRefusal( + ctx context.Context, operation, rdName string, +) (int, *apiv1.APICallRc) { + existing, err := s.Store.ResourceDefinitions().GetUncached(ctx, rdName) + if errors.Is(err, store.ErrNotFound) { + // The authoritative read says nothing stands under the name: the + // cached read that found the leftover was the stale one, and there is + // no mark to refuse over. + return 0, nil + } + if err != nil { - writeCloneRefused(w, http.StatusInternalServerError, srcName, cloneName, &apiv1.APICallRc{ + return http.StatusInternalServerError, &apiv1.APICallRc{ RetCode: apiCallRcError, - Message: "clone target '" + cloneName + "' exists, but reading it back to check " + - "for an abandoned rollback failed: " + err.Error(), + Message: operation + " target '" + rdName + "' exists, but reading it back to check " + + "for an abandoned rollback failed: " + scrubImplDetails(err.Error()), Cause: "an earlier attempt whose rollback gave up leaves a definition that looks " + "whole, and only the mark it carries tells the two apart", - Correc: "retry the clone", - }) + Correc: "retry the " + operation, + } + } - return true + // The DELETE flag is read here too, past the cache: the gates ahead of + // this one read it from a cache that may not have seen a definition + // deleted directly, and a definition being deleted is not one to answer + // for, whatever its replicas say. It comes before the rollback mark on + // purpose, as the cached DELETE gate in cloneTargetState does: a + // definition a rollback has flagged for deletion is waited out, not given + // the advice for a rollback that stopped. + if slices.Contains(existing.Flags, rdFlagDelete) { + return http.StatusConflict, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: operation + " target '" + rdName + "' is being deleted", + Correc: "wait for the delete to finish, then " + operation + " again", + } } spelled := existing.Props[rollbackAbandonedKey] if spelled == "" { - return false + return 0, nil + } + + // A rollback that has not reported a step is one still running, or one + // whose process stopped mid-way. Both are refused, but the first ends on + // its own, so the advice says so before it says to delete by hand; the + // CLI door words it the same way (store.JudgeRestoreLeftover). + if spelled == rollbackInProgress { + return http.StatusConflict, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: operation + " target '" + rdName + "' is being rolled back by the attempt " + + "that created it", + Cause: "an earlier attempt at this " + operation + " started rolling the definition " + + "back and has not reported how it ended: it is still running, or it stopped mid-way", + Correc: "retry the " + operation + " once that rollback has finished, which takes under a " + + "minute; if the mark is still there a minute after the failure, " + clearRollbackMarkCommand(rdName) + + " to keep '" + rdName + "' as it stands, or delete it by hand. Cleared while the rollback " + + "still runs, it stops that rollback before it deletes anything", + } } step, known := rollbackStepByName(spelled) - cause, correc := rollbackStepAdvice(step, known, cloneName) + cause, correc := rollbackStepAdvice(step, known, rdName) - writeCloneRefused(w, http.StatusConflict, srcName, cloneName, &apiv1.APICallRc{ + return http.StatusConflict, &apiv1.APICallRc{ RetCode: apiCallRcError, - Message: "clone target '" + cloneName + "' is what an earlier attempt left when its " + + Message: operation + " target '" + rdName + "' is what an earlier attempt left when its " + "rollback gave up (" + spelled + ")", - Cause: "an earlier attempt at this clone failed and could not be rolled back, so the " + - "definition may hold less than the clone intended; " + cause, + Cause: "an earlier attempt at this " + operation + " failed and could not be rolled back, " + + "so the definition may hold less than the " + operation + " intended; " + cause, Correc: correc, - }) - - return true -} - -// cloneLeftoverRefusal words the refusal for a leftover that is not whole, -// naming what is missing. -// -// None of these states proves the attempt behind it is dead. The marker lands -// before the volumes and the volumes before the replicas, so a first attempt -// still running looks exactly like an attempt that stopped, and the correction -// has to hold for both: an operator reading "delete it" must not be pointed at -// a clone that is about to finish. -func cloneLeftoverRefusal(cloneName string, state cloneLeftover) *apiv1.APICallRc { - const replayWouldLie = "; answering this retry as an idempotent replay would bind a volume " + - "to a clone that exists on no node" - - correcIfStopped := "if no clone under that name is still running, delete '" + cloneName + - "' by hand, then clone again" - - refusal := &apiv1.APICallRc{ - RetCode: apiCallRcError, - Message: "clone target '" + cloneName + "' exists but is not a whole clone", - Correc: correcIfStopped, - } - - switch state { - case cloneLeftoverTearingDown: - refusal.Message = "clone target '" + cloneName + "' exists but is still being torn down" - refusal.Cause = "the definition under that name carries this clone's marker, and every " + - "replica of it is already accepted for deletion" + replayWouldLie - refusal.Correc = "wait until the replicas of '" + cloneName + "' are gone, then clone again" - case cloneLeftoverEmpty: - refusal.Cause = "the definition under that name carries this clone's marker but has no " + - "volumes and no replicas: an attempt at this clone stopped before creating any " + - "volume, or has not reached that step yet" + replayWouldLie - case cloneLeftoverNoVolumes: - refusal.Cause = "the definition under that name carries this clone's marker and has " + - "replicas but no volumes" + replayWouldLie - case cloneLeftoverNoReplicas, cloneLeftoverWhole: - refusal.Cause = "the definition under that name carries this clone's marker and has " + - "volumes but no replicas: an attempt at this clone has not placed any yet, or a " + - "rollback removed them and could not remove the definition" + replayWouldLie - } - - return refusal + } } // ensureCloneSnapshot takes (or reuses) the internal snapshot backing @@ -783,31 +1439,39 @@ func cloneLeftoverRefusal(cloneName string, state cloneLeftover) *apiv1.APICallR // online, and every backing pool must be snapshot-capable (thin LVM // / ZFS / FILE_THIN) — the clone data plane IS a snapshot restore, // so a source that cannot be snapshotted cannot be cloned. -func (s *Server) ensureCloneSnapshot(w http.ResponseWriter, r *http.Request, src *apiv1.ResourceDefinition, cloneName string) (*apiv1.Snapshot, bool) { +func (s *Server) ensureCloneSnapshot( + w http.ResponseWriter, r *http.Request, src *apiv1.ResourceDefinition, cloneName string, +) (*apiv1.Snapshot, bool, bool) { ctx := r.Context() snapName := cloneSnapshotName(cloneName) - existing, err := s.Store.Snapshots().Get(ctx, src.Name, snapName) - if err == nil { - // Interrupted-clone retry: the snapshot landed on a previous - // attempt; reuse it so the restore sees the same point-in-time. - return &existing, true + existing, found, ok := readCloneSnapshot(ctx, w, s.Store, src.Name, cloneName) + if !ok { + return nil, false, false + } + + if found { + if !s.cloneSnapshotIsReusable(ctx, w, src, &existing, cloneName) { + return nil, false, false + } + + return &existing, true, true } snap := apiv1.Snapshot{Name: snapName, ResourceName: src.Name} - err = s.hydrateSnapshotFromRD(ctx, &snap, src.Name) + err := s.hydrateSnapshotFromRD(ctx, &snap, src.Name) if err != nil { writeCloneRefused(w, http.StatusInternalServerError, src.Name, cloneName, &apiv1.APICallRc{ RetCode: apiCallRcError, - Message: "clone of resource definition '" + src.Name + "' failed: " + err.Error(), + Message: "clone of resource definition '" + src.Name + "' failed: " + scrubImplDetails(err.Error()), }) - return nil, false + return nil, false, false } if !s.cloneSnapshotPreconditionsHold(ctx, w, src, &snap, cloneName) { - return nil, false + return nil, false, false } snap.Snapshots = makeSnapshotPerNode(snapName, snap.Nodes, snap.VolumeDefinitions) @@ -817,13 +1481,387 @@ func (s *Server) ensureCloneSnapshot(w http.ResponseWriter, r *http.Request, src writeCloneRefused(w, http.StatusInternalServerError, src.Name, cloneName, &apiv1.APICallRc{ RetCode: apiCallRcError, Message: "clone of resource definition '" + src.Name + - "' failed: internal snapshot create: " + err.Error(), + "' failed: internal snapshot create: " + scrubImplDetails(err.Error()), }) - return nil, false + return nil, false, false + } + + return &snap, false, true +} + +// cloneSourceIsNotBeingDeleted mirrors the snapshot-create Bug 180 gate: a +// source RD mid-tear-down would reap the internal snapshot and the clone +// marker from under the satellite's restore. +func (s *Server) cloneSourceIsNotBeingDeleted( + ctx context.Context, w http.ResponseWriter, src *apiv1.ResourceDefinition, cloneName string, +) bool { + if !rdHasDeleteFlag(ctx, s, src.Name) { + return true + } + + writeCloneRefused(w, http.StatusConflict, src.Name, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + src.Name + "' refused: the source is being deleted", + Cause: "the source RD carries the DELETE flag; its backing data is being torn down", + Correc: "clone before deleting the source, or restore from a snapshot taken earlier", + }) + + return false +} + +// replayOfFinishedClone answers a retry of a clone that already completed, +// before anything on this path reads the live source. It returns +// (replayed, halt): replayed when the success has been written, halt when a +// refusal has. +// +// A replay of a clone that already completed has to survive anything that +// happened to the source since: it is a copy of a point-in-time and owes +// the source nothing. Every question asked ahead of it is a way to refuse +// one — the layer stack, the DELETE flag, and the shape comparison, whose +// "shape" is derived from the live source, so an ordinary +// `rd modify --resource-group` on the source turned every later replay +// into a 409 with a correction the caller it is aimed at cannot follow: +// linstor-csi sends the same body every time. +// +// Taking the internal snapshot is worse than a refusal. When +// `clone-` has been deleted, the create branch snapshots the CURRENT +// source before anyone asks whether the clone is done, so a pure replay +// mutates cluster state and, once the source has grown, leaves a fresh +// snapshot that diverges from the finished target — and refuses the +// replay from then on, permanently, with a correction that destroys a +// clone holding live data. +// +// The marker and the DELETE flag are read here only to decide whether this is +// even the right question. A leftover that is somebody else's definition, or +// one being torn down, is left to cloneTargetState, which owns those refusals +// and their wording. +func (s *Server) replayOfFinishedClone( + ctx context.Context, w http.ResponseWriter, srcName string, req *rdCloneRequest, +) (bool, bool) { + cloneName := req.Name + + existing, err := s.Store.ResourceDefinitions().Get(ctx, cloneName) + if errors.Is(err, store.ErrNotFound) { + return false, false + } + + if err != nil { + writeCloneTargetUnreadable(w, srcName, cloneName, err) + + return false, true + } + + if !restoreMarkerMatches(existing.Props, srcName, cloneSnapshotName(cloneName)) || + slices.Contains(existing.Flags, rdFlagDelete) { + return false, false + } + + // Hoisting this check above everything that reads the live source must + // not also hoist it above everything that reads the REQUEST. A replay + // naming a resource group or a layer stack the finished clone does not + // have would otherwise be told the clone completed and handed the other + // shape — a caller who named LUKS gets plaintext. The comparison is + // against the leftover, never the source. linstor-csi names both fields + // on every clone, taken from the StorageClass, so its replay names the + // shape the first request stamped on the leftover and passes. + // + // It runs before anything established whether the leftover is finished, + // so its wording is the resume's: the leftover was started with a shape, + // whether or not it got further. + if differs := requestedShapeDiffers(&existing, req.ResourceGroup, req.LayerList); differs != "" { + writeCloneRefused(w, http.StatusConflict, srcName, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone target '" + cloneName + "' was started with " + differs, + Cause: "a retry keeps the definition an earlier attempt created, so the shape " + + "this request asks for would be validated and then ignored", + Correc: "retry with the shape the clone was started with, or delete '" + + cloneName + "' and clone again", + }) + + return false, true + } + + finished, halt := s.cloneLeftoverIsFinished(ctx, w, srcName, cloneName) + if halt || !finished { + return false, halt + } + + if !s.cloneLeftoverIsUsable(ctx, w, srcName, &existing) { + return false, true + } + + if !s.answerFinishedClone(ctx, w, srcName, cloneName, existing.Name, req) { + return false, true + } + + return true, false +} + +// answerFinishedClone answers the replay of a finished clone: it takes the +// adoption mark, applies the request's prop edits, and reports 201. False +// means a refusal has been written instead. +func (s *Server) answerFinishedClone( + ctx context.Context, w http.ResponseWriter, srcName, cloneName, storedName string, req *rdCloneRequest, +) bool { + if status, refusal := s.finishedLeftoverRefusal(ctx, "clone", storedName); refusal != nil { + writeCloneRefused(w, status, srcName, cloneName, refusal) + + return false + } + + // The prop edits still have to land. A replay carrying override_props / + // delete_props / delete_namespaces would otherwise be answered 201 with + // them dropped, which is the accept-and-drop this endpoint refuses + // external_name and volume_passphrases to avoid — and it is reachable + // without any caller changing their mind, because the first attempt can + // fail in applyClonePropEdits AFTER the volumes are already there. The + // edits are a patch, so re-applying what already landed is what makes the + // replay idempotent rather than a second write. + err := s.applyClonePropEdits(ctx, req) + if err != nil { + writeCloneRefused(w, http.StatusInternalServerError, srcName, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + srcName + "' is complete, but " + + "applying override_props/delete_props failed: " + scrubImplDetails(err.Error()), + }) + + return false + } + + writeCloneStarted(w, srcName, cloneName, cloneDoneMessage(true, cloneName), nil) + + return true +} + +// cloneProgress is where a target carrying a clone's or a restore's marker +// stands; see store.LeftoverProgress. The replay and the status poll both ask +// it, and they have to get the same answer: a POST told 201 followed by a GET +// that is not COMPLETE for the same clone, one request apart, is a successful +// clone the driver then waits on. +type cloneProgress = store.LeftoverProgress + +const ( + cloneUnfinished = store.LeftoverUnfinished + cloneFinished = store.LeftoverFinished + cloneForeign = store.LeftoverForeign + cloneTearingDown = store.LeftoverTearingDown +) + +// assessMarkedClone classifies a target that carries this clone's marker. It +// reads and never writes, so the poll can call it as freely as the replay. +// +// Volumes alone do not decide it. materializeRestoredRD hydrates the volumes +// and THEN stamps the replicas, so a shell holding every volume and no replica +// is an ordinary intermediate state, and one whose data exists nowhere but the +// snapshot. +// +// Where the replicas are is not part of the question. A clone that has one +// replica holds its data, and which nodes carry it afterwards is placement, +// which ordinary operations change: evacuating a node moves a replica off a +// node the snapshot recorded, and scaling down removes one. Asking whether +// every snapshot node still carries a replica read both as unfinished, and the +// resume then re-stamped a replica on the node the operator had just emptied, +// restored from the point-in-time while the surviving replica had moved on +// with live writes: two replicas of one definition with different content. A +// clone left short of its placement by a first attempt that died between two +// stamps is the same state seen from outside, and topping it up is the +// placement reconciliation's job, not the clone's: linstor-csi v1.10.1 calls +// reconcileResourcePlacement right after its COMPLETE loop in +// pkg/client/linstor.go. +// +// Nothing here reads the live source. A finished clone is a copy of a +// point-in-time, so it is judged against the snapshot it was restored from +// while that exists, and against itself once it does not: an absent snapshot +// cannot unmake a clone that has its volumes and a replica, and re-taking one +// to answer the question is the write a replay must not make. "Could not read +// the snapshot" is not "the snapshot is gone", and only the second takes the +// face-value branch. +// +// The volumes and replicas are read past the cache, the status poll included, +// at two API-server reads per poll: a poll that read a stale cache could report +// COMPLETE over a replica already being torn down, and linstor-csi binds the +// volume on that answer without asking again. +func assessMarkedClone( + ctx context.Context, st store.Store, srcName, cloneName string, +) (cloneProgress, error) { + targetVDs, err := store.LiveVolumes(ctx, st, cloneName) + if err != nil { + return cloneUnfinished, errors.Wrapf(err, "list the volumes of %q", cloneName) + } + + if len(targetVDs) == 0 { + return assessLeftover(ctx, st, cloneName, nil, nil, true) + } + + snap, err := st.Snapshots().Get(ctx, srcName, cloneSnapshotName(cloneName)) + if err != nil && !errors.Is(err, store.ErrNotFound) { + return cloneUnfinished, errors.Wrapf(err, "read the snapshot behind %q", cloneName) + } + + var captured *apiv1.Snapshot + if err == nil { + captured = &snap + } + + return assessLeftover(ctx, st, cloneName, targetVDs, captured, true) +} + +// assessLeftover is store.AssessLeftover, the one judgement every door that +// resumes a marker-bearing leftover shares: the clone through +// assessMarkedClone, the REST restore through restoreLeftoverIsFinished, and +// the CLI restore. +func assessLeftover( + ctx context.Context, st store.Store, targetName string, + targetVDs []apiv1.VolumeDefinition, snap *apiv1.Snapshot, needReplica bool, +) (cloneProgress, error) { + return store.AssessLeftover(ctx, st, targetName, targetVDs, snap, needReplica) //nolint:wrapcheck // wrapped by store with the target +} + +// countReplicas is store.CountReplicas. +func countReplicas(ctx context.Context, st store.Store, rdName string) (store.ReplicaCount, error) { + return store.CountReplicas(ctx, st, rdName) //nolint:wrapcheck // wrapped by store with the definition +} + +// cloneLeftoverIsFinished answers assessMarkedClone for the replay, writing the +// refusal it implies. It returns (finished, stop). +func (s *Server) cloneLeftoverIsFinished( + ctx context.Context, w http.ResponseWriter, srcName, cloneName string, +) (bool, bool) { + progress, err := assessMarkedClone(ctx, s.Store, srcName, cloneName) + if err != nil { + writeCloneRefused(w, http.StatusInternalServerError, srcName, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + srcName + "' failed: " + scrubImplDetails(err.Error()), + }) + + return false, true + } + + switch progress { + case cloneFinished: + return true, false + case cloneForeign: + writeCloneRefused(w, http.StatusConflict, srcName, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + srcName + "' refused: '" + cloneName + + "' holds a volume this clone would not have written", + Cause: "the definition under that name carries volumes that are not the ones " + + "this clone restores, and hydrating skips what is already there, so the " + + "retry would leave them and report the clone complete", + Correc: "delete '" + cloneName + "' and clone again, or clone under a different name", + }) + + return false, true + case cloneTearingDown: + writeCloneRefused(w, http.StatusConflict, srcName, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone target '" + cloneName + "' exists but is still being torn down", + Cause: "the definition under that name carries this clone's marker, and every " + + "replica of it is already accepted for deletion; answering this retry as an " + + "idempotent replay would bind a volume to a clone that exists on no node", + Correc: "wait until the replicas of '" + cloneName + "' are gone, then clone again", + }) + + return false, true + case cloneUnfinished: + } + + return false, false +} + +// cloneSnapshotIsCurrent refuses to resume a clone over a leftover snapshot +// that no longer describes the source. False means a refusal has been written. +// +// "Found" is not "still right". The first attempt takes `clone-` and +// dies; the source is resized, or gains a volume; the retry hydrates the +// target from the stale snapshot and answers 201, and the clone-status poll +// then reports COMPLETE because the volume counts agree. The caller is handed +// a clone at the old shape with nothing saying so. This only became reachable +// when the retry started resuming instead of reporting the leftover done. +// +// Refusing rather than retaking is the call `blockstor rd clone` already makes +// (internal/cli/definition.go): the snapshot may be the only copy of +// something, and deleting it is the operator's decision, not this endpoint's. +func (s *Server) cloneSnapshotIsCurrent( + ctx context.Context, w http.ResponseWriter, + src *apiv1.ResourceDefinition, snap *apiv1.Snapshot, cloneName string, +) bool { + current, err := store.LiveVolumes(ctx, s.Store, src.Name) + if err != nil { + writeCloneRefused(w, http.StatusInternalServerError, src.Name, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + src.Name + "' failed: " + scrubImplDetails(err.Error()), + }) + + return false + } + + divergence := snapshotDivergence(src.Name, snap, current) + if divergence == "" { + return true + } + + // The correction has to be one that works. When the leftover already holds + // volumes hydrated from this snapshot, deleting only the snapshot makes the + // next attempt retake it at the new size and collide with those volumes — + // a bare 500 with no correction at all, over a fresh snapshot of the live + // source now claiming to be this clone's origin. Only then does the retry + // reach the refusal that names the target. Say so on the first answer. + correc := "delete the snapshot '" + snap.Name + "' so the clone retakes it, " + + "or clone under a different name" + + hydrated, listErr := store.LiveVolumes(ctx, s.Store, cloneName) + if listErr == nil && len(hydrated) > 0 { + correc = "delete '" + cloneName + "' and the snapshot '" + snap.Name + + "', then clone again, or clone under a different name" + } + + writeCloneRefused(w, http.StatusConflict, src.Name, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "clone of resource definition '" + src.Name + "' refused: " + divergence, + Cause: "an earlier attempt at this clone left the snapshot '" + snap.Name + + "' behind and the source has changed since; resuming from it would " + + "materialise the clone at the old shape and report it complete", + Correc: correc, + }) + + return false +} + +// snapshotDivergence describes how a snapshot has fallen behind the volumes it +// was taken of, or "" while it still matches. +// +// Sizes are compared keyed by volume number, since neither list promises an +// order — and a resize leaves the count alone, so counting volumes answers +// only half the question. +func snapshotDivergence(rdName string, snap *apiv1.Snapshot, current []apiv1.VolumeDefinition) string { + if len(current) != len(snap.VolumeDefinitions) { + return snap.Name + " covers " + strconv.Itoa(len(snap.VolumeDefinitions)) + + " volume(s) but " + rdName + " now has " + strconv.Itoa(len(current)) + } + + captured := make(map[int32]int64, len(snap.VolumeDefinitions)) + for _, vol := range snap.VolumeDefinitions { + captured[vol.VolumeNumber] = vol.SizeKib + } + + for i := range current { + was, ok := captured[current[i].VolumeNumber] + if !ok { + return snap.Name + " does not cover volume " + + strconv.FormatInt(int64(current[i].VolumeNumber), 10) + " of " + rdName + } + + if was != current[i].SizeKib { + return snap.Name + " captured volume " + + strconv.FormatInt(int64(current[i].VolumeNumber), 10) + " at " + + strconv.FormatInt(was, 10) + " KiB but " + rdName + " is now " + + strconv.FormatInt(current[i].SizeKib, 10) + " KiB" + } } - return &snap, true + return "" } // cloneSnapshotPreconditionsHold runs the snapshot-feasibility guards @@ -851,6 +1889,22 @@ func (s *Server) cloneSnapshotPreconditionsHold(ctx context.Context, w http.Resp return false } + return s.cloneSnapshotNodesAreUsable(ctx, w, src, snap, cloneName) +} + +// cloneSnapshotNodesAreUsable is the half of the preconditions that a REUSED +// snapshot has to satisfy as well as a freshly taken one. +// +// The nodes recorded on the snapshot are where the restore places the clone's +// replicas, so an offline node or a pool that cannot hold a snapshot is the +// same problem whichever attempt took it. Running these only on the branch +// that creates the snapshot gave one cluster state two answers: the first +// clone refused with 503, the retry over the leftover snapshot answered 201 +// and stamped a replica on a node whose satellite cannot act on it. +func (s *Server) cloneSnapshotNodesAreUsable( + ctx context.Context, w http.ResponseWriter, + src *apiv1.ResourceDefinition, snap *apiv1.Snapshot, cloneName string, +) bool { if offline := s.offlineTargetNodes(ctx, snap.Nodes); len(offline) > 0 { writeCloneRefused(w, http.StatusServiceUnavailable, src.Name, cloneName, &apiv1.APICallRc{ RetCode: apiCallRcError, @@ -878,7 +1932,7 @@ func (s *Server) clonePoolsSupportSnapshots(ctx context.Context, w http.Response if err != nil { writeCloneRefused(w, http.StatusInternalServerError, src.Name, cloneName, &apiv1.APICallRc{ RetCode: apiCallRcError, - Message: "clone of resource definition '" + src.Name + "' failed: " + err.Error(), + Message: "clone of resource definition '" + src.Name + "' failed: " + scrubImplDetails(err.Error()), }) return false @@ -908,7 +1962,7 @@ func (s *Server) clonePoolsSupportSnapshots(ctx context.Context, w http.Response // parity with the empty-shell path, which folds them in during the // shallow copy. No-op when the request carries neither. func (s *Server) applyClonePropEdits(ctx context.Context, req *rdCloneRequest) error { - if len(req.OverrideProps) == 0 && len(req.DeleteProps) == 0 { + if len(req.OverrideProps) == 0 && len(req.DeleteProps) == 0 && len(req.DeleteNamespaces) == 0 { return nil } @@ -928,22 +1982,67 @@ func (s *Server) applyClonePropEdits(ctx context.Context, req *rdCloneRequest) e delete(rd.Props, k) } + deletePropNamespaces(rd.Props, req.DeleteNamespaces) + return nil }) } -// writeCloneRefused stamps a clone refusal in the CloneStarted OBJECT -// envelope. python-linstor's `resource_dfn_clone` decodes the -// response body into CloneStarted unconditionally (success AND -// error), so every non-2xx answer on the clone POST must keep the -// object shape — a bare `[]ApiCallRc` array crashes the CLI with -// `AttributeError: 'list' object has no attribute 'get'` before the -// error line reaches the operator (see writeSnapshotCloneNotImplemented). +// cloneEnvelope carries, for one clone request, which shape its refusals are +// written in. +// +// The two clients that clone decode a refusal in shapes that exclude each +// other. python-linstor's `resource_dfn_clone` decodes every answer, success +// and error alike, into CloneStarted, and a bare `[]ApiCallRc` crashes it with +// `AttributeError: 'list' object has no attribute 'get'` before the error line +// reaches the operator. golinstor's do() (client/client.go) decodes every +// non-2xx answer other than a 404 as `[]ApiCallRc`, so the object reached +// linstor-csi as `json: cannot unmarshal object into Go value of type +// client.ApiCallError`, and the cause and correction never reached anyone. +// +// python-linstor is the one client that needs the object, and it names itself +// on every request: `PythonLinstor/ (API)` (linstorapi.py, +// python-linstor 1.27.1). Its refusals keep the object. Every other caller +// gets the array the rest of the API answers with: linstor-csi, which sends +// `linstor-csi/`, and any other golinstor client, which may send no +// name of its own at all and decodes errors the same way. A 404 carries +// nothing to a golinstor client either way: it maps every 404 to its bare +// NotFoundError without reading the body. +// +// The trade-off is deliberate. A third-party client written against +// upstream's object shape, and naming itself something other than +// python-linstor, now gets the array. None is known; every client this API is +// built for is one of the two above, and the array is the shape the rest of +// the API answers errors with, so it is the default that fails least. +type cloneEnvelope struct { + http.ResponseWriter + + array bool +} + +// pythonLinstorUserAgentPrefix is how python-linstor names itself to the API. +const pythonLinstorUserAgentPrefix = "PythonLinstor/" + +func withCloneEnvelope(w http.ResponseWriter, r *http.Request) http.ResponseWriter { + return &cloneEnvelope{ + ResponseWriter: w, + array: !strings.HasPrefix(r.UserAgent(), pythonLinstorUserAgentPrefix), + } +} + +// writeCloneRefused stamps a clone refusal in the envelope the caller decodes; +// see cloneEnvelope. func writeCloneRefused(w http.ResponseWriter, status int, srcName, cloneName string, callRc *apiv1.APICallRc) { if callRc.ObjRefs == nil { callRc.ObjRefs = map[string]string{objRefRscDfn: srcName} } + if envelope, ok := w.(*cloneEnvelope); ok && envelope.array { + writeJSON(w, status, []apiv1.APICallRc{*callRc}) + + return + } + writeJSON(w, status, cloneStartedResponse{ Location: "/v1/resource-definitions/" + srcName + "/clone/" + cloneName, SourceName: srcName, @@ -952,6 +2051,15 @@ func writeCloneRefused(w http.ResponseWriter, status int, srcName, cloneName str }) } +// writeCloneStoreError is writeStoreError in the envelope this endpoint +// answers in. A bare []ApiCallRc crashes python-linstor's clone decode, so a +// typo in the source name, or a replay of a volume-less clone, lost its message +// to an AttributeError. +func writeCloneStoreError(w http.ResponseWriter, srcName, cloneName string, err error) { + status, callRc := storeErrorRc(err, storeKindResourceDfn) + writeCloneRefused(w, status, srcName, cloneName, &callRc) +} + // cloneEmptyRDShell materialises the empty-source clone path: shallow-copy // of the RD spec (Props, RG ref) under a new name. Group D's integration // smoke test pins this branch — a freshly-created vol-less RD must be @@ -969,9 +2077,35 @@ func writeCloneRefused(w http.ResponseWriter, status int, srcName, cloneName str func (s *Server) cloneEmptyRDShell(w http.ResponseWriter, r *http.Request, src *apiv1.ResourceDefinition, req *rdCloneRequest, ) { + // The same gate the data path has. A volume-less source inside its delete + // window has nothing to lose, but the shell below is a copy of it, and a + // definition born carrying DELETE is refused by every snapshot, restore and + // clone that later names it, each saying to wait for a delete that never + // comes. + if !s.cloneSourceIsNotBeingDeleted(r.Context(), w, src, req.Name) { + return + } + clone := *src clone.Name = req.Name clone.UUID = "" + // Flags are the source's lifecycle, not part of its shape: a shell that + // inherited them would carry a delete, or any later flag, it never had. + clone.Flags = nil + + // The caller's own shape wins over the source's, on both clone paths. + // Without one the shell is stamped with the source's data-plane stack, as + // the data path stamps its target: left empty under a target group of + // another stack, the control plane would judge it by that group while the + // satellite brings it up as the default, once volumes are added. + clone.LayerStack = store.EffectiveLayerStack(src.LayerStack) + if len(req.LayerList) > 0 { + clone.LayerStack = req.LayerList + } + + if req.ResourceGroup != "" { + clone.ResourceGroupName = req.ResourceGroup + } if src.Props != nil || len(req.OverrideProps) > 0 { clone.Props = make(map[string]string, len(src.Props)+len(req.OverrideProps)) @@ -984,9 +2118,11 @@ func (s *Server) cloneEmptyRDShell(w http.ResponseWriter, r *http.Request, delete(clone.Props, k) } + deletePropNamespaces(clone.Props, req.DeleteNamespaces) + err := s.Store.ResourceDefinitions().Create(r.Context(), &clone) if err != nil { - writeStoreError(w, err) + writeCloneStoreError(w, src.Name, clone.Name, err) return } @@ -1006,42 +2142,131 @@ func (s *Server) cloneEmptyRDShell(w http.ResponseWriter, r *http.Request, writeCloneStarted(w, src.Name, clone.Name, "resource definition cloned: "+clone.Name, uncheckedRG) } -// handleRDCloneStatus answers golinstor's `CloneStatus` poll. The -// response is grounded in actual store state (Bug 114): we compare -// the source RD's VolumeDefinition count to the target's. Equal -// counts → COMPLETE (the clone is structurally consistent with the -// source). A non-empty source paired with an empty target → FAILED, -// so linstor-csi surfaces a concrete error rather than spinning on -// a stale COMPLETE while the data plane never copied anything. +// handleRDCloneStatus answers golinstor's `CloneStatus` poll, grounded +// in actual store state (Bug 114). A target carrying this clone's marker +// is answered by answerMarkedCloneStatus; one without it by +// computeCloneStatus's volume-count comparison. // // Path: GET /v1/resource-definitions/{src}/clone/{target}. -// A 404 on the target signals "clone failed mid-way" — which gives -// linstor-csi an actionable error rather than an infinite poll loop. -// A 404 on the source surfaces the same way: it would have been -// caught at clone-POST time, but a delete-source race shouldn't -// produce a phantom COMPLETE either. +// A 404 on the target signals "no finished clone under that name", +// which linstor-csi acts on by issuing the clone, rather than an +// infinite poll loop. func (s *Server) handleRDCloneStatus(w http.ResponseWriter, r *http.Request) { srcName := r.PathValue("rd") targetName := r.PathValue("target") - _, err := s.Store.ResourceDefinitions().Get(r.Context(), targetName) + target, err := s.Store.ResourceDefinitions().Get(r.Context(), targetName) if err != nil { writeStoreError(w, err) return } - status := computeCloneStatus(r.Context(), s.Store, srcName, targetName) + if restoreMarkerMatches(target.Props, srcName, cloneSnapshotName(targetName)) { + s.answerMarkedCloneStatus(r.Context(), w, srcName, &target) + + return + } + writeJSON(w, http.StatusOK, client.ResourceDefinitionCloneStatus{ - Status: status, + Status: computeCloneStatus(r.Context(), s.Store, srcName, targetName), }) } -// computeCloneStatus resolves COMPLETE vs FAILED for a clone pair by -// comparing source-vs-target VolumeDefinition counts. Bug 114: an -// empty target paired with a non-empty source is structurally -// incomplete — golinstor's poll loop must see FAILED so it stops -// waiting on data that will never arrive. +// answerMarkedCloneStatus answers the poll for a target that went through the +// snapshot data plane. It is judged the way the replay judges it, against its +// own point-in-time and never against the live source: comparing the live +// source's volume count told FAILED to the poll that follows a 201 replay the +// moment anyone added a volume to the source, which is legal and routine. +// +// Only a finished clone is answered with a status. Everything else is answered +// in the one shape its caller can act on. linstor-csi POSTs the clone only +// when this GET is a 404, and then polls until COMPLETE with no FAILED branch +// and no second POST (pkg/client/linstor.go, v1.10.1), so FAILED over a +// leftover the POST knows how to resume left the driver waiting on it forever, +// and so did CLONING. A 404 sends it back to the POST, which resumes an +// unfinished leftover or refuses a foreign one with a cause and a correction, +// written in the array linstor-csi decodes so they reach the PVC's events (see +// cloneEnvelope). A read failure is a 500 for the same reason: +// the driver returns it from CreateVolume and retries, where FAILED would bind +// nothing and end nothing. +// +// COMPLETE over a clone whose placement is short of the snapshot's nodes is +// safe: the driver reconciles placement right after it. +// +// A definition carrying the DELETE flag is torn down whatever its replicas say, +// the way the POST's cloneTargetState refuses it: one deleted directly rather +// than through `rd d`, which cascades the replicas first, still lists a live +// replica, and its replicas alone would read it complete. +// +// A finished one is then put through the gate the POST's replay asks before it +// answers 201 (cloneLeftoverRefusal): a parent group that is gone, a rollback +// in progress or one that gave up. The driver binds on COMPLETE without ever +// reaching the POST, so a COMPLETE over a clone the RG-deleted rollback is +// deleting binds a volume that is about to be gone. This narrows that race and +// does not close it: a poll that reads the group from a cache that has not +// seen its delete, before the rollback's mark lands, still answers COMPLETE, +// and nothing the poll does is visible to the rollback. It is the same window +// as any `rg d` racing a clone. +func (s *Server) answerMarkedCloneStatus( + ctx context.Context, w http.ResponseWriter, srcName string, target *apiv1.ResourceDefinition, +) { + targetName := target.Name + + progress := cloneTearingDown + + var err error + if !slices.Contains(target.Flags, rdFlagDelete) { + progress, err = assessMarkedClone(ctx, s.Store, srcName, targetName) + } + + if err != nil { + writeJSON(w, http.StatusInternalServerError, []apiv1.APICallRc{{ + RetCode: apiCallRcError, + Message: "clone status of '" + targetName + "' from '" + srcName + "' could not be read: " + scrubImplDetails(err.Error()), + }}) + + return + } + + cause := "an earlier attempt at this clone stopped before it finished" + + switch progress { + case cloneFinished: + status, refusal := s.cloneLeftoverRefusal(ctx, target) + if refusal == nil { + writeJSON(w, http.StatusOK, client.ResourceDefinitionCloneStatus{Status: clonestatus.Complete}) + + return + } + + if status == http.StatusInternalServerError { + writeJSON(w, status, []apiv1.APICallRc{*refusal}) + + return + } + + cause = refusal.Message + case cloneForeign: + cause = "the definition under that name holds a volume this clone would not have written" + case cloneTearingDown: + cause = "the definition under that name, or every replica of it, is being torn down" + case cloneUnfinished: + } + + writeJSON(w, http.StatusNotFound, []apiv1.APICallRc{{ + RetCode: apiCallRcError | apiCallRcFailNotFoundRscDfn, + Message: "no finished clone '" + targetName + "' of '" + srcName + "'", + Cause: cause, + Correc: "issue the clone again: it resumes what the earlier attempt left, or says why it cannot", + }}) +} + +// computeCloneStatus resolves COMPLETE vs FAILED for a clone pair whose target +// carries no marker, a clone made before the snapshot data plane existed, by +// comparing source-vs-target VolumeDefinition counts. Bug 114: an empty target +// paired with a non-empty source is structurally incomplete — golinstor's poll +// loop must see FAILED so it stops waiting on data that will never arrive. // // If the source RD itself is gone (race with `rd d ` while the // poll is in flight), we cannot prove the target is consistent — @@ -1080,27 +2305,22 @@ func computeCloneStatus(ctx context.Context, st store.Store, srcName, targetName // 501 so the operator either learns the gap immediately or scripts // the snapshot+restore fallback. func writeSnapshotCloneNotImplemented(w http.ResponseWriter, srcName, cloneName, srcSnapName string) { - writeJSON(w, http.StatusNotImplemented, cloneStartedResponse{ - Location: "/v1/resource-definitions/" + srcName + "/clone/" + cloneName, - SourceName: srcName, - CloneName: cloneName, - Messages: &[]apiv1.APICallRc{{ - RetCode: apiCallRcError, - Message: "snapshot-based clone not implemented in this release (pending Phase 12)", - Cause: "the apiserver accepts `src_snap_name` on the wire for python-linstor 1.27.0 " + - "compatibility (Bug 232 + 237) but the satellite-side snapshot-clone data plane is " + - "not yet wired; silently falling back to a live-RD shell copy would discard the " + - "snapshot intent and produce a clone with the wrong contents", - Correc: "use the snapshot-then-restore workflow which IS wired today: " + - "`linstor s create " + srcName + " " + srcSnapName + "` (if the snapshot " + - "doesn't already exist) then " + - "`linstor s resource restore --from-resource " + srcName + - " --from-snapshot " + srcSnapName + " --to-resource " + cloneName + "`", - ObjRefs: map[string]string{ - objRefRscDfn: srcName, - "SnapName": srcSnapName, - }, - }}, + writeCloneRefused(w, http.StatusNotImplemented, srcName, cloneName, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "snapshot-based clone not implemented in this release (pending Phase 12)", + Cause: "the apiserver accepts `src_snap_name` on the wire for python-linstor 1.27.0 " + + "compatibility (Bug 232 + 237) but the satellite-side snapshot-clone data plane is " + + "not yet wired; silently falling back to a live-RD shell copy would discard the " + + "snapshot intent and produce a clone with the wrong contents", + Correc: "use the snapshot-then-restore workflow which IS wired today: " + + "`linstor s create " + srcName + " " + srcSnapName + "` (if the snapshot " + + "doesn't already exist) then " + + "`linstor s resource restore --from-resource " + srcName + + " --from-snapshot " + srcSnapName + " --to-resource " + cloneName + "`", + ObjRefs: map[string]string{ + objRefRscDfn: srcName, + "SnapName": srcSnapName, + }, }) } diff --git a/pkg/rest/rd_clone_client_envelope_test.go b/pkg/rest/rd_clone_client_envelope_test.go new file mode 100644 index 00000000..43a41ac6 --- /dev/null +++ b/pkg/rest/rd_clone_client_envelope_test.go @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "strings" + "testing" + + lapi "github.com/LINBIT/golinstor/client" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +func linstorCSIClient(t *testing.T, base string) *lapi.Client { + t.Helper() + + u, err := url.Parse(base) + if err != nil { + t.Fatalf("parse base: %v", err) + } + + c, err := lapi.NewClient(lapi.BaseURL(u), lapi.UserAgent("linstor-csi/v1.11.2")) + if err != nil { + t.Fatalf("golinstor NewClient: %v", err) + } + + return c +} + +// golinstor decodes every non-2xx answer other than a 404 as []ApiCallRc, so +// the CloneStarted object reached linstor-csi as a JSON decode error and the +// cause and correction of every clone refusal were lost before the PVC's +// events. +func TestRDCloneRefusalsReachLinstorCSIAsApiCallErrors(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-env") + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "dst-taken"}); err != nil { + t.Fatalf("seed the colliding definition: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + _, err := linstorCSIClient(t, base).ResourceDefinitions.Clone(ctx, "src-env", + lapi.ResourceDefinitionCloneRequest{Name: "dst-taken"}) + + var apiErr lapi.ApiCallError + if !errors.As(err, &apiErr) || len(apiErr) == 0 { + t.Fatalf("clone refusal reached golinstor as %T %v, want an ApiCallError", err, err) + } + + if !strings.Contains(apiErr[0].Message, "dst-taken") || apiErr[0].Correction == "" { + t.Errorf("refusal %+v lost its message or correction", apiErr[0]) + } +} + +// A golinstor client other than linstor-csi decodes errors the same way, and +// may not name itself at all: golinstor sends no User-Agent of its own unless +// asked. It gets the array too. +func TestRDCloneRefusalsReachAnyGolinstorClientAsApiCallErrors(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-goenv") + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "dst-gotaken"}); err != nil { + t.Fatalf("seed the colliding definition: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + u, err := url.Parse(base) + if err != nil { + t.Fatalf("parse base: %v", err) + } + + c, err := lapi.NewClient(lapi.BaseURL(u)) + if err != nil { + t.Fatalf("golinstor NewClient: %v", err) + } + + _, err = c.ResourceDefinitions.Clone(ctx, "src-goenv", lapi.ResourceDefinitionCloneRequest{Name: "dst-gotaken"}) + + var apiErr lapi.ApiCallError + if !errors.As(err, &apiErr) || len(apiErr) == 0 { + t.Fatalf("clone refusal reached a plain golinstor client as %T %v, want an ApiCallError", err, err) + } +} + +// python-linstor decodes every clone answer into CloneStarted and names itself +// on every request, so its refusals keep the object. +func TestRDCloneRefusalsStayObjectsForPythonLinstor(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-pyenv") + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "dst-pytaken"}); err != nil { + t.Fatalf("seed the colliding definition: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, base+"/v1/resource-definitions/src-pyenv/clone", + strings.NewReader(`{"name":"dst-pytaken"}`)) + if err != nil { + t.Fatalf("build the request: %v", err) + } + + req.Header.Set("User-Agent", pythonLinstorTestUserAgent) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("POST clone: %v", err) + } + + defer func() { _ = resp.Body.Close() }() + + raw, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read the body: %v", err) + } + + var envelope cloneStartedResponse + if err := json.Unmarshal(raw, &envelope); err != nil || envelope.Messages == nil || len(*envelope.Messages) == 0 { + t.Errorf("refusal to python-linstor is not a CloneStarted object with messages: %s", raw) + } +} diff --git a/pkg/rest/rd_clone_csi_retry_test.go b/pkg/rest/rd_clone_csi_retry_test.go new file mode 100644 index 00000000..1d826854 --- /dev/null +++ b/pkg/rest/rd_clone_csi_retry_test.go @@ -0,0 +1,485 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "context" + "encoding/json" + "net/http" + "slices" + "strings" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// csiPollBudget bounds cloneLikeCSI's poll loop, which in the driver has none. +const csiPollBudget = 5 + +// cloneLikeCSI drives the clone the way linstor-csi v1.10.1 does +// (pkg/client/linstor.go): GET the status, POST the clone only when that GET +// is a 404, then poll until COMPLETE. The driver has no FAILED branch and no +// second POST, so any answer other than COMPLETE after the POST is a wait that +// never ends; here it is a bounded loop that reports what it saw. +func cloneLikeCSI(t *testing.T, base, src, dst string) (bool, string) { + t.Helper() + + statusURL := base + "/v1/resource-definitions/" + src + "/clone/" + dst + + code, status := cloneStatusOnce(t, statusURL) + if code == http.StatusNotFound { + resp := postClone(t, base, src, map[string]any{ + "name": dst, "use_zfs_clone": true, + "layer_list": []string{"DRBD", "STORAGE"}, "resource_group": "DfltRscGrp", + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + return false, "POST answered " + http.StatusText(resp.StatusCode) + } + + code, status = cloneStatusOnce(t, statusURL) + } + + for range csiPollBudget { + if code != http.StatusOK { + return false, "poll answered " + http.StatusText(code) + } + + if status == "COMPLETE" { + return true, status + } + + code, status = cloneStatusOnce(t, statusURL) + } + + return false, "poll never left " + status +} + +func cloneStatusOnce(t *testing.T, url string) (int, string) { + t.Helper() + + resp := httpGet(t, url) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return resp.StatusCode, "" + } + + var got struct { + Status string `json:"status"` + } + + if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + t.Fatalf("decode clone-status: %v", err) + } + + return resp.StatusCode, got.Status +} + +func seedDefaultGroup(t *testing.T, st store.Store) { + t.Helper() + + if err := st.ResourceGroups().Create(t.Context(), &apiv1.ResourceGroup{Name: "DfltRscGrp"}); err != nil && + !strings.Contains(err.Error(), "exist") { + t.Fatalf("seed DfltRscGrp: %v", err) + } +} + +func seedOnlineNodeWithPool(t *testing.T, st store.Store, node string) { + t.Helper() + + ctx := t.Context() + + if err := st.Nodes().Create(ctx, &apiv1.Node{Name: node, ConnectionStatus: "ONLINE"}); err != nil { + t.Fatalf("seed %s: %v", node, err) + } + + if err := st.StoragePools().Create(ctx, &apiv1.StoragePool{ + StoragePoolName: "zfs-thin", + NodeName: node, + ProviderKind: "ZFS_THIN", + SupportsSnapshot: true, + }); err != nil { + t.Fatalf("seed pool on %s: %v", node, err) + } +} + +// Evacuating a node moves a clone's replica off a node the snapshot recorded. +// The replay then read the clone as unfinished and re-stamped a replica on the +// node the operator had just emptied, restored from the point-in-time while the +// migrated replica had moved on with live writes. +func TestRDCloneReplayLeavesAMigratedReplicaWhereItIs(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-mig") + seedOnlineNodeWithPool(t, st, "node-c") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-mig", "dst-mig", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + if err := st.Resources().Create(ctx, &apiv1.Resource{ + Name: "dst-mig", NodeName: "node-c", Props: map[string]string{"StorPoolName": "zfs-thin"}, + }); err != nil { + t.Fatalf("place the replica on node-c: %v", err) + } + + if err := st.Resources().Delete(ctx, "dst-mig", "node-a"); err != nil { + t.Fatalf("evacuate node-a: %v", err) + } + + if code := cloneOnce(t, base, "src-mig", "dst-mig", nil); code != http.StatusCreated { + t.Fatalf("replay after migration = %d, want 201", code) + } + + after, err := st.Resources().ListByDefinition(ctx, "dst-mig") + if err != nil { + t.Fatalf("list the clone's replicas: %v", err) + } + + if names := nodeNamesOf(after); !slices.Equal(names, []string{"node-c"}) { + t.Errorf("replicas after the replay = %v, want [node-c]", names) + } + + if code, status := cloneStatusOnce(t, base+"/v1/resource-definitions/src-mig/clone/dst-mig"); status != "COMPLETE" { + t.Errorf("poll after migration = %d %q, want COMPLETE", code, status) + } +} + +// The status poll answered FAILED over a leftover the POST knows how to resume, +// and linstor-csi POSTs only after a 404 and polls with no FAILED branch, so +// the leftover was never resumed and the driver waited on it forever. +func TestRDCloneStatusSendsAnUnfinishedCloneBackToThePost(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-csi") + seedDefaultGroup(t, st) + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-csi", "dst-csi", map[string]any{ + "layer_list": []string{"DRBD", "STORAGE"}, "resource_group": "DfltRscGrp", + }); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + // The first attempt died after hydrating and before any replica held the + // data: every volume, no replica. + if err := st.Resources().Delete(ctx, "dst-csi", "node-a"); err != nil { + t.Fatalf("drop the replica: %v", err) + } + + if done, saw := cloneLikeCSI(t, base, "src-csi", "dst-csi"); !done { + t.Fatalf("the driver's clone flow over an unfinished leftover did not complete: %s", saw) + } + + after, err := st.Resources().ListByDefinition(ctx, "dst-csi") + if err != nil { + t.Fatalf("list the clone's replicas: %v", err) + } + + if len(after) == 0 { + t.Error("COMPLETE was reached without the resume placing a replica") + } +} + +// A foreign target is refused by the POST with a cause and a correction. The +// poll has to send the driver there too, rather than answer a status it waits +// on forever. +func TestRDCloneStatusSendsAForeignTargetToTheRefusal(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-for") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-for", "dst-for", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + vd, err := st.VolumeDefinitions().Get(ctx, "dst-for", 0) + if err != nil { + t.Fatalf("read the clone's volume: %v", err) + } + + vd.SizeKib /= 2 + if err := st.VolumeDefinitions().Update(ctx, "dst-for", &vd); err != nil { + t.Fatalf("shrink the volume: %v", err) + } + + code, status := cloneStatusOnce(t, base+"/v1/resource-definitions/src-for/clone/dst-for") + if code != http.StatusNotFound { + t.Errorf("poll over a foreign target = %d %q, want 404 so the driver reaches the POST's refusal", code, status) + } +} + +// "Could not read" is not an answer about the clone. FAILED on a read failure +// was a wait without end; a 500 is an error the driver returns and retries. +func TestRDCloneStatusIsAnErrorWhenTheCloneCannotBeRead(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-err") + + base, stop := startServerWithStore(t, backend) + + if code := cloneOnce(t, base, "src-err", "dst-err", nil); code != http.StatusCreated { + stop() + t.Fatalf("first clone = %d, want 201", code) + } + + stop() + + base2, stop2 := startServerWithStore(t, failingSnapshotGetStore{backend}) + defer stop2() + + code, status := cloneStatusOnce(t, base2+"/v1/resource-definitions/src-err/clone/dst-err") + if code != http.StatusInternalServerError { + t.Errorf("poll over an unreadable snapshot = %d %q, want 500", code, status) + } +} + +// The classifier calls a volume larger than the snapshot recorded this clone's +// own, expanded since; the hydrate the resume routes into refused any size that +// was not equal, so the same leftover was admitted by one half of the state +// machine and answered a bare 500 by the other, on every retry. +func TestRDCloneResumeOfAnExpandedCloneWithNoReplicaCompletes(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-grow") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-grow", "dst-grow", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + vd, err := st.VolumeDefinitions().Get(ctx, "dst-grow", 0) + if err != nil { + t.Fatalf("read the clone's volume: %v", err) + } + + grown := vd.SizeKib * 2 + + vd.SizeKib = grown + if err := st.VolumeDefinitions().Update(ctx, "dst-grow", &vd); err != nil { + t.Fatalf("expand the clone: %v", err) + } + + if err := st.Resources().Delete(ctx, "dst-grow", "node-a"); err != nil { + t.Fatalf("drop the replica: %v", err) + } + + if code := cloneOnce(t, base, "src-grow", "dst-grow", nil); code != http.StatusCreated { + t.Fatalf("retry over an expanded leftover with no replica = %d, want 201", code) + } + + after, err := st.VolumeDefinitions().Get(ctx, "dst-grow", 0) + if err != nil { + t.Fatalf("read the clone's volume after the retry: %v", err) + } + + if after.SizeKib != grown { + t.Errorf("volume size after the retry = %d, want the expanded %d", after.SizeKib, grown) + } +} + +// materializeRestoredRD stamps the source's group on the clone when the request +// names none, so a real leftover carries one. A request that names no group has +// to be compared as naming nothing, not as naming "", or a source moved to +// another group refuses every later retry of the clone it already made. +func TestRDCloneRetryNamingNoGroupPassesALeftoverThatCarriesOne(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + unfinish bool + }{ + {name: "finished", unfinish: false}, + {name: "unfinished", unfinish: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + src, dst := "src-rg6-"+tc.name, "dst-rg6-"+tc.name + seedDeployedCloneSource(t, st, src) + + for _, group := range []string{"grp-a", "grp-b"} { + if err := st.ResourceGroups().Create(ctx, &apiv1.ResourceGroup{Name: group}); err != nil { + t.Fatalf("seed %s: %v", group, err) + } + } + + moveSourceToGroup(t, st, src, "grp-a") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, src, dst, nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + leftover, err := st.ResourceDefinitions().Get(ctx, dst) + if err != nil { + t.Fatalf("read the clone: %v", err) + } + + if leftover.ResourceGroupName != "grp-a" { + t.Fatalf("fixture: the clone carries group %q, want grp-a", leftover.ResourceGroupName) + } + + if tc.unfinish { + if err := st.Resources().Delete(ctx, dst, "node-a"); err != nil { + t.Fatalf("drop the replica: %v", err) + } + } + + moveSourceToGroup(t, st, src, "grp-b") + + if code := cloneOnce(t, base, src, dst, nil); code != http.StatusCreated { + t.Errorf("retry naming no group over a leftover in grp-a = %d, want 201", code) + } + }) + } +} + +func moveSourceToGroup(t *testing.T, st store.Store, rdName, group string) { + t.Helper() + + rd, err := st.ResourceDefinitions().Get(t.Context(), rdName) + if err != nil { + t.Fatalf("read %s: %v", rdName, err) + } + + rd.ResourceGroupName = group + if err := st.ResourceDefinitions().Update(t.Context(), &rd); err != nil { + t.Fatalf("move %s to %s: %v", rdName, group, err) + } +} + +// The shape refusal runs before anything established whether the leftover is +// finished, so it must not tell the operator that it is. +func TestRDCloneShapeRefusalDoesNotCallAnUnfinishedLeftoverFinished(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-word") + seedCloneLeftover(t, st, "src-word", "dst-word") + + if err := st.ResourceGroups().Create(ctx, &apiv1.ResourceGroup{Name: "grp-other"}); err != nil { + t.Fatalf("seed RG: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-word", map[string]any{ + "name": "dst-word", "use_zfs_clone": true, "resource_group": "grp-other", + }) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409", resp.StatusCode) + } + + envelope := decodeCloneStarted(t, resp) + if envelope.Messages == nil || len(*envelope.Messages) == 0 { + t.Fatal("empty envelope") + } + + msg := (*envelope.Messages)[0] + if text := msg.Message + " " + msg.Cause; strings.Contains(text, "finished") { + t.Errorf("refusal over a leftover with no volumes says it is finished: %q", text) + } +} + +// The replay is asked before every check that reads the live source, and each +// of those is a way to refuse a clone that already completed. Only the snapshot +// step was pinned; these two hold the others. +func TestRDCloneReplayIsAskedBeforeTheSourceChecks(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + change func(t *testing.T, st store.Store, src string) + }{ + { + name: "source-being-deleted", + change: func(t *testing.T, st store.Store, src string) { + t.Helper() + updateSource(t, st, src, func(rd *apiv1.ResourceDefinition) { + rd.Flags = append(rd.Flags, rdFlagDelete) + }) + }, + }, + { + name: "source-stack-changed", + change: func(t *testing.T, st store.Store, src string) { + t.Helper() + updateSource(t, st, src, func(rd *apiv1.ResourceDefinition) { + rd.LayerStack = []string{"STORAGE"} + }) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + src, dst := "src-ord6-"+tc.name, "dst-ord6-"+tc.name + seedDeployedCloneSource(t, st, src) + seedDefaultGroup(t, st) + + base, stop := startServerWithStore(t, st) + defer stop() + + body := map[string]any{"layer_list": []string{"DRBD", "STORAGE"}, "resource_group": "DfltRscGrp"} + + if code := cloneOnce(t, base, src, dst, body); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + tc.change(t, st, src) + + if code := cloneOnce(t, base, src, dst, body); code != http.StatusCreated { + t.Errorf("replay of the finished clone after %s = %d, want 201", tc.name, code) + } + }) + } +} + +func updateSource(t *testing.T, st store.Store, rdName string, mutate func(*apiv1.ResourceDefinition)) { + t.Helper() + + ctx := context.WithoutCancel(t.Context()) + + rd, err := st.ResourceDefinitions().Get(ctx, rdName) + if err != nil { + t.Fatalf("read %s: %v", rdName, err) + } + + mutate(&rd) + + if err := st.ResourceDefinitions().Update(ctx, &rd); err != nil { + t.Fatalf("update %s: %v", rdName, err) + } +} diff --git a/pkg/rest/rd_clone_golinstor_shape_test.go b/pkg/rest/rd_clone_golinstor_shape_test.go new file mode 100644 index 00000000..a36689b0 --- /dev/null +++ b/pkg/rest/rd_clone_golinstor_shape_test.go @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "encoding/json" + "net/http" + "strings" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// seedCloneRG puts a resource group in the store. A clone that pins one by +// name is refused when it is not there, so every test that exercises +// resource_group has to create it first — the same order an operator works in. +func seedCloneRG(t *testing.T, st store.Store, names ...string) { + t.Helper() + + for _, name := range names { + if err := st.ResourceGroups().Create(t.Context(), &apiv1.ResourceGroup{Name: name}); err != nil { + t.Fatalf("seed RG %q: %v", name, err) + } + } +} + +// linstor-csi defaults LayerList to [DRBD, STORAGE] and never sends a clone +// without it, so a field this endpoint does not declare is a 400 on every +// clone-from-volume regardless of how the StorageClass is written. The body +// is decoded with DisallowUnknownFields, so the refusal lands before any of +// the handler runs. +func TestRDCloneAcceptsTheFullGolinstorBody(t *testing.T) { + st := store.NewInMemory() + if err := st.ResourceDefinitions().Create(t.Context(), + &apiv1.ResourceDefinition{Name: "pvc-src", ResourceGroupName: "grp"}); err != nil { + t.Fatalf("seed: %v", err) + } + + seedCloneRG(t, st, "grp") + + base, stop := startServerWithStore(t, st) + defer stop() + + // Exactly what golinstor marshals for linstor-csi v1.11.2's clone call + // (pkg/client/linstor.go): the name, the StorageClass's layer list and + // group, and the provisioning-completed prop to drop. It never sends + // use_zfs_clone. + body := []byte(`{"name":"pvc-dst","layer_list":["DRBD","STORAGE"],` + + `"resource_group":"grp","delete_props":["Aux/csi-provisioning-completed-by"]}`) + + resp := httpPost(t, base+"/v1/resource-definitions/pvc-src/clone", body) + defer func() { _ = resp.Body.Close() }() + + // Asserted as "created", not merely "not 400". Accepting anything + // outside 400 lets a 500 through, and this test is the one that would + // have to catch the endpoint accepting the body and then failing on it. + if resp.StatusCode != http.StatusCreated { + var raw map[string]any + + _ = json.NewDecoder(resp.Body).Decode(&raw) + + t.Fatalf("clone status = %d, want 201: %+v", resp.StatusCode, raw) + } +} + +// Declaring a field is only half of it. layer_list and resource_group are +// the caller's choice of shape, so a clone that ignores them hands back a +// definition built to the source's shape while reporting success. +func TestRDCloneHonoursLayerListAndResourceGroup(t *testing.T) { + st := store.NewInMemory() + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{ + Name: "pvc-src", + ResourceGroupName: "source-grp", + LayerStack: []string{"DRBD", "STORAGE"}, + }); err != nil { + t.Fatalf("seed: %v", err) + } + + seedCloneRG(t, st, "chosen-grp") + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(map[string]any{ + "name": "pvc-dst", + "layer_list": []string{"STORAGE"}, + "resource_group": "chosen-grp", + }) + + resp := httpPost(t, base+"/v1/resource-definitions/pvc-src/clone", body) + _ = resp.Body.Close() + + got, err := st.ResourceDefinitions().Get(t.Context(), "pvc-dst") + if err != nil { + t.Fatalf("get the clone: %v", err) + } + + if got.ResourceGroupName != "chosen-grp" { + t.Errorf("resource group = %q, want the one the caller asked for", got.ResourceGroupName) + } + + if len(got.LayerStack) != 1 || got.LayerStack[0] != "STORAGE" { + t.Errorf("layer stack = %v, want the one the caller asked for", got.LayerStack) + } +} + +// The two fields this endpoint cannot honour are refused rather than dropped. +// Accepting external_name and returning a definition under a different name, +// or accepting volume_passphrases and materialising volumes with keys the +// caller does not hold, is the failure mode src_snap_name is already refused +// for. +func TestRDCloneRefusesWhatItCannotHonour(t *testing.T) { + for name, tc := range map[string]struct{ body, field string }{ + "external name": {`{"name":"pvc-dst","external_name":"something-else"}`, "external_name"}, + "volume passphrases": {`{"name":"pvc-dst","volume_passphrases":["k1"]}`, "volume_passphrases"}, + // python-linstor drops `name` from the body when it sends + // external_name; the refusal still names the field it cannot honour. + "external name, python-linstor body": {`{"external_name":"something-else"}`, "external_name"}, + } { + st := store.NewInMemory() + if err := st.ResourceDefinitions().Create(t.Context(), + &apiv1.ResourceDefinition{Name: "pvc-src"}); err != nil { + t.Fatalf("%s: seed: %v", name, err) + } + + base, stop := startServerWithStore(t, st) + + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, + base+"/v1/resource-definitions/pvc-src/clone", strings.NewReader(tc.body)) + if err != nil { + t.Fatalf("%s: build the request: %v", name, err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "PythonLinstor/1.27.0") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%s: post: %v", name, err) + } + + rc := decodeCloneMessage(t, resp) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusNotImplemented { + t.Errorf("%s: status = %d, want 501", name, resp.StatusCode) + } + + if !strings.Contains(rc.Message, tc.field) { + t.Errorf("%s: refusal %q does not name %s", name, rc.Message, tc.field) + } + + if _, err := st.ResourceDefinitions().Get(t.Context(), "pvc-dst"); err == nil { + t.Errorf("%s: the refused clone was created anyway", name) + } + + stop() + } +} + +// The same on the path CSI actually takes. A source with volumes clones +// through the snapshot-restore machinery, which builds the target from the +// SOURCE's shape — so the caller's layer_list and resource_group have to be +// carried into it, not just applied on the volume-less shortcut. +func TestRDCloneHonoursTheCallersShapeOnTheDataPath(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-shape") + seedCloneRG(t, st, "chosen-grp") + + base, stop := startServerWithStore(t, st) + defer stop() + + // The layer stack has to be the source's on this path — the clone + // restores bytes and brings the stack up over them — so what is honoured + // here is the group, and the stack is sent the way linstor-csi sends it. + resp := postClone(t, base, "src-shape", map[string]any{ + "name": "dst-shape", + "layer_list": apiv1.DefaultLayerStack(), + "resource_group": "chosen-grp", + "use_zfs_clone": true, + }) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("clone status got %d, want 201", resp.StatusCode) + } + + dst, err := st.ResourceDefinitions().Get(ctx, "dst-shape") + if err != nil { + t.Fatalf("target RD not persisted: %v", err) + } + + if dst.ResourceGroupName != "chosen-grp" { + t.Errorf("resource group = %q, want the one the caller asked for", dst.ResourceGroupName) + } + + if len(dst.LayerStack) != len(apiv1.DefaultLayerStack()) { + t.Errorf("layer stack = %v, want the one the caller asked for", dst.LayerStack) + } +} diff --git a/pkg/rest/rd_clone_idempotency_test.go b/pkg/rest/rd_clone_idempotency_test.go new file mode 100644 index 00000000..bd58039c --- /dev/null +++ b/pkg/rest/rd_clone_idempotency_test.go @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "encoding/json" + "net/http" + "strings" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// seedCloneLeftover puts under the target name what a first clone attempt +// leaves behind when it dies after creating the definition and before +// hydrating the volumes: the definition, the marker, no volumes. +func seedCloneLeftover(t *testing.T, st store.Store, srcName, cloneName string, flags ...string) { + t.Helper() + + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{ + Name: cloneName, + Flags: flags, + Props: map[string]string{ + restoreFromSnapshotKey: restoreMarker(srcName, cloneSnapshotName(cloneName)), + }, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } +} + +// The marker is stamped with the definition, before the volumes are hydrated +// and the replicas placed, so a target carrying it says a clone STARTED — not +// that one finished. Answering 201 on the marker alone turns the retry +// linstor-csi issues after a partial failure into a silent incomplete: CSI +// sees the volume as ready and nothing ever finishes it. +// +// The status is 201 either way, which is the point: only the volumes tell the +// resumed clone from the one that merely agreed it had already happened. +func TestRDCloneResumesAnIncompleteLeftover(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-resume") + seedCloneLeftover(t, st, "src-resume", "dst-resume") + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-resume", map[string]any{ + "name": "dst-resume", + "use_zfs_clone": true, + }) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("retry over an incomplete leftover: got %d, want 201", resp.StatusCode) + } + + vds, err := st.VolumeDefinitions().List(t.Context(), "dst-resume") + if err != nil { + t.Fatalf("list the target's volumes: %v", err) + } + + if len(vds) != 1 { + t.Fatalf("after the retry the target has %d volume(s), want 1 — the clone "+ + "reported success over a definition it never finished", len(vds)) + } + + // And it says so, rather than reading like a first run. + var envelope cloneStartedResponse + if err := json.NewDecoder(resp.Body).Decode(&envelope); err != nil { + t.Fatalf("decode the clone envelope: %v", err) + } + + if envelope.Messages == nil || len(*envelope.Messages) == 0 { + t.Fatal("the retry returned an empty envelope") + } + + if !strings.Contains((*envelope.Messages)[0].Message, "on retry") { + t.Errorf("message = %q, want it to name the retry", (*envelope.Messages)[0].Message) + } +} + +// Resuming stops at a leftover being torn down: finishing it would race the +// deletion reaping the very objects the clone writes. +func TestRDCloneRefusesALeftoverBeingDeleted(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-dying") + seedCloneLeftover(t, st, "src-dying", "dst-dying", rdFlagDelete) + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-dying", map[string]any{ + "name": "dst-dying", + "use_zfs_clone": true, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409 — that leftover is being deleted", resp.StatusCode) + } + + vds, err := st.VolumeDefinitions().List(t.Context(), "dst-dying") + if err != nil { + t.Fatalf("list the leftover's volumes: %v", err) + } + + if len(vds) != 0 { + t.Errorf("the refused clone hydrated %d volume(s) into a definition being deleted", len(vds)) + } +} + +// A definition under the target name that this clone did not create stays a +// refusal, and stays untouched. +func TestRDCloneStillRefusesAForeignTarget(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-foreign") + + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{ + Name: "dst-foreign", + Props: map[string]string{"someone": "else"}, + }); err != nil { + t.Fatalf("seed the occupant: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-foreign", map[string]any{"name": "dst-foreign"}) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409", resp.StatusCode) + } + + got, err := st.ResourceDefinitions().Get(t.Context(), "dst-foreign") + if err != nil { + t.Fatalf("get the occupant: %v", err) + } + + if got.Props["someone"] != "else" { + t.Error("the refused clone overwrote the definition that was already there") + } +} + +// resource_group lands on the target on both clone paths — the shallow copy +// stamps it, the data path hands it to materializeRestoredRD — and neither +// went past the validator RD-create runs. A typo produced a clone whose parent +// group does not exist: it lists fine and places badly, because the placer's +// Controller→RG→RD prop walk drops the RG tier without a word. +func TestRDCloneRefusesAnUnknownResourceGroup(t *testing.T) { + t.Parallel() + + for name, src := range map[string]func(t *testing.T, st store.Store){ + "volume-less source": func(t *testing.T, st store.Store) { + t.Helper() + + if err := st.ResourceDefinitions().Create(t.Context(), + &apiv1.ResourceDefinition{Name: "src-rg"}); err != nil { + t.Fatalf("seed: %v", err) + } + }, + "source with volumes": func(t *testing.T, st store.Store) { + t.Helper() + seedDeployedCloneSource(t, st, "src-rg") + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + src(t, st) + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-rg", map[string]any{ + "name": "dst-rg", + "resource_group": "no-such-group", + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status = %d, want 404 — that resource group does not exist", + resp.StatusCode) + } + + if _, err := st.ResourceDefinitions().Get(t.Context(), "dst-rg"); err == nil { + t.Error("the refused clone was created anyway, with a dangling parent group") + } + + // Refused before anything is written: without the pre-check the + // data path snapshots the source and materialises the target, + // and the post-write guard rolls the target back to the same + // 404 while the internal snapshot stays on the source. + if _, err := st.Snapshots().Get(t.Context(), "src-rg", cloneSnapshotName("dst-rg")); err == nil { + t.Error("the refused clone left its internal snapshot on the source") + } + }) + } +} + +// The positive control: the same request with the group actually there. +func TestRDCloneHonoursAKnownResourceGroup(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + if err := st.ResourceDefinitions().Create(t.Context(), + &apiv1.ResourceDefinition{Name: "src-rg-ok"}); err != nil { + t.Fatalf("seed: %v", err) + } + + if err := st.ResourceGroups().Create(t.Context(), + &apiv1.ResourceGroup{Name: "real-group"}); err != nil { + t.Fatalf("seed RG: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-rg-ok", map[string]any{ + "name": "dst-rg-ok", + "resource_group": "real-group", + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status = %d, want 201", resp.StatusCode) + } + + got, err := st.ResourceDefinitions().Get(t.Context(), "dst-rg-ok") + if err != nil { + t.Fatalf("get the clone: %v", err) + } + + if got.ResourceGroupName != "real-group" { + t.Errorf("resource group = %q, want the one the caller asked for", got.ResourceGroupName) + } +} + +// delete_namespaces rides with override_props and delete_props on every +// upstream props-modify body. Declaring its two neighbours and not it left a +// clone body carrying delete_namespaces on the very 400 declaring the other two +// was meant to end. +func TestRDCloneHonoursDeleteNamespaces(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{ + Name: "src-ns", + Props: map[string]string{ + "DrbdOptions": "bare", + "DrbdOptions/Net/protocol": "C", + "DrbdOptionsOther": "keep", + "Aux/keep": "yes", + }, + }); err != nil { + t.Fatalf("seed: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-ns", map[string]any{ + "name": "dst-ns", + "delete_namespaces": []string{"DrbdOptions"}, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status = %d, want 201 — delete_namespaces is on every "+ + "props-modify body upstream sends", resp.StatusCode) + } + + got, err := st.ResourceDefinitions().Get(t.Context(), "dst-ns") + if err != nil { + t.Fatalf("get the clone: %v", err) + } + + for _, key := range []string{"DrbdOptions/Net/protocol"} { + if _, present := got.Props[key]; present { + t.Errorf("prop %q survived the namespace delete", key) + } + } + + // The namespace covers the keys below it: the key spelled as it, and one + // that merely starts like it, are different keys, as upstream reads them. + for _, key := range []string{"DrbdOptions", "DrbdOptionsOther", "Aux/keep"} { + if _, present := got.Props[key]; !present { + t.Errorf("prop %q was deleted, and it is outside the named namespace", key) + } + } +} diff --git a/pkg/rest/rd_clone_layer_stack_test.go b/pkg/rest/rd_clone_layer_stack_test.go new file mode 100644 index 00000000..a077589f --- /dev/null +++ b/pkg/rest/rd_clone_layer_stack_test.go @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "net/http" + "slices" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// seedCloneSourceWithStack is seedDeployedCloneSource with a layer stack put +// on the source, which is what a clone's requested stack is compared against. +func seedCloneSourceWithStack(t *testing.T, st store.Store, rdName string, stack []string) { + t.Helper() + + seedDeployedCloneSource(t, st, rdName) + + rd, err := st.ResourceDefinitions().Get(t.Context(), rdName) + if err != nil { + t.Fatalf("read the seeded source: %v", err) + } + + rd.LayerStack = stack + + if err := st.ResourceDefinitions().Update(t.Context(), &rd); err != nil { + t.Fatalf("put a layer stack on the source: %v", err) + } +} + +// The clone data plane restores the source's bytes and brings the layer stack +// up over them, in that order, and every layer's bring-up writes to the device +// it is handed. LUKS: luks.Format treats a device carrying no LUKS header as +// one to format, so adding it does not hand back an encrypted copy — it +// formats away the data just restored and reports COMPLETE. DRBD: create-md +// runs with --force over `meta-disk internal`, stamping metadata across the +// tail of the same bytes, and the only gate before it looks for DRBD metadata +// rather than for a filesystem. +// +// Both directions of both layers are refused: a dropped layer leaves the +// target reading data the missing layer wrote. +func TestRDCloneRefusesALayerStackChange(t *testing.T) { + t.Parallel() + + plain := []string{"DRBD", "STORAGE"} + encrypted := []string{"DRBD", "LUKS", "STORAGE"} + bare := []string{"STORAGE"} + + for name, tc := range map[string]struct { + sourceStack []string + requested []string + }{ + "adding LUKS to a plaintext source": {sourceStack: plain, requested: encrypted}, + "dropping LUKS from an encrypted source": {sourceStack: encrypted, requested: plain}, + // DRBD writes to the device as surely as LUKS does: create-md runs + // with --force over `meta-disk internal`, stamping metadata across + // the tail of the bytes the clone just restored, and the only gate + // before it looks for DRBD metadata rather than for a filesystem. + "adding DRBD to a source without it": {sourceStack: bare, requested: plain}, + "dropping DRBD from a source with it": {sourceStack: plain, requested: bare}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedCloneSourceWithStack(t, st, "src-luks", tc.sourceStack) + + // The cluster passphrase is set, so the refusal under test is the + // membership change and not the LUKS-prereq gate that runs above it. + base, stop := startServerWithPassphrase(t, st) + defer stop() + + resp := postClone(t, base, "src-luks", map[string]any{ + "name": "dst-luks", + "layer_list": tc.requested, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", resp.StatusCode) + } + + if _, err := st.ResourceDefinitions().Get(t.Context(), "dst-luks"); err == nil { + t.Error("the refused clone was created anyway") + } + + // The refusal has to land before the data plane starts, not after: + // the internal snapshot is the first thing the clone writes. + if _, err := st.Snapshots().Get(t.Context(), "src-luks", + cloneSnapshotName("dst-luks")); err == nil { + t.Error("the refused clone snapshotted the source anyway") + } + }) + } +} + +// The positive control for the refusals above. Same request shape, same +// passphrase, a layer_list spelled in a case the source is not stored in, and +// the clone goes through, because the set is the same. (The order is the +// stack's own and validateLayerStack enforces it, so case is what is free to +// differ.) Without this the refusals could be coming from validateLayerStack or +// from the LUKS-prereq gate and the table above would not know the difference. +func TestRDCloneHonoursALayerListThatMatchesTheSource(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedCloneSourceWithStack(t, st, "src-keep", []string{"DRBD", "LUKS", "STORAGE"}) + + base, stop := startServerWithPassphrase(t, st) + defer stop() + + resp := postClone(t, base, "src-keep", map[string]any{ + "name": "dst-keep", + "layer_list": []string{"drbd", "luks", "storage"}, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status = %d, want 201", resp.StatusCode) + } + + dst, err := st.ResourceDefinitions().Get(t.Context(), "dst-keep") + if err != nil { + t.Fatalf("target RD not persisted: %v", err) + } + + // Stored in canonical case: the controller asks whether a definition + // carries DRBD by exact match, and a lowercase stack read as one without. + if !slices.Equal(dst.LayerStack, []string{"DRBD", "LUKS", "STORAGE"}) { + t.Errorf("layer stack = %v, want [DRBD LUKS STORAGE]", dst.LayerStack) + } +} + +// The CSI hot path, which is the one this gate must never touch. linstor-csi +// sends [DRBD, STORAGE] on every clone; a definition created without an +// explicit stack stores none, and reading that as "no layers" would make the +// request read as adding both and refuse every clone-from-volume — the defect +// this PR exists to fix, reintroduced by its own guard. An unset stack means +// the default, here as everywhere else that reads one. +func TestRDCloneOfASourceWithNoRecordedStackTakesTheCSIDefault(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-bare") + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-bare", map[string]any{ + "name": "dst-bare", + "layer_list": apiv1.DefaultLayerStack(), + "use_zfs_clone": true, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status = %d, want 201 — this is what linstor-csi sends on every clone", + resp.StatusCode) + } + + if _, err := st.ResourceDefinitions().Get(t.Context(), "dst-bare"); err != nil { + t.Errorf("target RD not persisted: %v", err) + } +} + +// The empty-source path carries no bytes to lose, so it keeps taking the +// caller's stack whatever it says about LUKS. Refusing there would break the +// only reason the endpoint accepts layer_list at all. +func TestRDCloneOfAVolumelessSourceStillTakesAnyStack(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{ + Name: "src-empty", + LayerStack: []string{"DRBD", "STORAGE"}, + }); err != nil { + t.Fatalf("seed: %v", err) + } + + base, stop := startServerWithPassphrase(t, st) + defer stop() + + resp := postClone(t, base, "src-empty", map[string]any{ + "name": "dst-empty", + "layer_list": []string{"DRBD", "LUKS", "STORAGE"}, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status = %d, want 201", resp.StatusCode) + } + + dst, err := st.ResourceDefinitions().Get(t.Context(), "dst-empty") + if err != nil { + t.Fatalf("target RD not persisted: %v", err) + } + + if !apiv1.LayerInStack(dst.LayerStack, apiv1.LayerKindLUKS) { + t.Errorf("layer stack = %v, want the LUKS stack the caller asked for", dst.LayerStack) + } +} diff --git a/pkg/rest/rd_clone_leftover_shape_test.go b/pkg/rest/rd_clone_leftover_shape_test.go new file mode 100644 index 00000000..165f936c --- /dev/null +++ b/pkg/rest/rd_clone_leftover_shape_test.go @@ -0,0 +1,671 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "context" + "net/http" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// growSourceVolume resizes the source's volume 0, the routine operation that +// must not change what a finished clone's retry answers. +func growSourceVolume(t *testing.T, st store.Store, rdName string, sizeKib int64) { + t.Helper() + + vd, err := st.VolumeDefinitions().Get(t.Context(), rdName, 0) + if err != nil { + t.Fatalf("read the source volume: %v", err) + } + + vd.SizeKib = sizeKib + + if err := st.VolumeDefinitions().Update(t.Context(), rdName, &vd); err != nil { + t.Fatalf("grow the source: %v", err) + } +} + +// A clone that COMPLETED is a copy of a point-in-time and owes the source +// nothing afterwards. linstor-csi replays CreateVolume whenever a response is +// lost or external-provisioner restarts, so the replay has to be the +// idempotent success CSI requires. +// +// Comparing the leftover snapshot against the live source made an ordinary +// resize poison every later retry, permanently: the internal snapshot is +// deterministic and outlives the clone by design, the marker survives, and so +// every repeat took the same refusal — one whose correction, "delete the +// snapshot", the operator cannot follow, because that snapshot is the origin +// of the clone they already have. +func TestRDCloneReplayOfAFinishedCloneSurvivesASourceResize(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-replay") + + base, stop := startServerWithStore(t, st) + defer stop() + + body := map[string]any{"name": "dst-replay", "use_zfs_clone": true} + + first := postClone(t, base, "src-replay", body) + _ = first.Body.Close() + + if first.StatusCode != http.StatusCreated { + t.Fatalf("first clone: got %d, want 201", first.StatusCode) + } + + // The clone is finished. Now the source grows, which has nothing to do + // with the copy already taken. + growSourceVolume(t, st, "src-replay", 128*1024) + + replay := postClone(t, base, "src-replay", body) + _ = replay.Body.Close() + + if replay.StatusCode != http.StatusCreated { + t.Fatalf("replay after a source resize: got %d, want 201 — a finished clone is a "+ + "copy of a point-in-time, and CSI replays this call whenever a response is lost", + replay.StatusCode) + } + + // And the replay left the finished clone alone, at the size it was taken. + vd, err := st.VolumeDefinitions().Get(ctx, "dst-replay", 0) + if err != nil { + t.Fatalf("read the clone's volume: %v", err) + } + + if vd.SizeKib != 64*1024 { + t.Errorf("clone volume 0 = %d KiB, want the 65536 it was cloned at", vd.SizeKib) + } +} + +// A leftover target holding volumes that are NOT the ones this clone would +// restore is not a resume. Hydrating skips what is already there, so the retry +// would leave the old shape in place and report the clone complete — and the +// status poll compares volume counts, not sizes, so it would agree. +func TestRDCloneRefusesALeftoverTargetWithADifferentVolumeShape(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-shape") + + // The snapshot an earlier attempt took, at the source's current size. + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: cloneSnapshotName("dst-shape"), + ResourceName: "src-shape", + Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + }); err != nil { + t.Fatalf("seed the leftover snapshot: %v", err) + } + + seedCloneLeftover(t, st, "src-shape", "dst-shape") + + // ... and a volume already under that name at the wrong size. + if err := st.VolumeDefinitions().Create(ctx, "dst-shape", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 8 * 1024}); err != nil { + t.Fatalf("seed the stale volume: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-shape", map[string]any{ + "name": "dst-shape", + "use_zfs_clone": true, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409 — that target holds a volume this clone would not "+ + "have written, and hydrating would skip it and report success", resp.StatusCode) + } + + vd, err := st.VolumeDefinitions().Get(ctx, "dst-shape", 0) + if err != nil { + t.Fatalf("read the target's volume: %v", err) + } + + if vd.SizeKib != 8*1024 { + t.Errorf("the refused clone rewrote the volume to %d KiB", vd.SizeKib) + } +} + +// The create branch refuses a source that would place no replicas (Bug 114). +// The reuse branch ran none of those checks, so a leftover snapshot recording +// no nodes placed nothing and the clone answered 201 over an empty shell. +func TestRDCloneRefusesALeftoverSnapshotWithNoNodes(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-nonodes") + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: cloneSnapshotName("dst-nonodes"), + ResourceName: "src-nonodes", + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + }); err != nil { + t.Fatalf("seed the node-less snapshot: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-nonodes", map[string]any{ + "name": "dst-nonodes", + "use_zfs_clone": true, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409 — a snapshot with no nodes places no replicas", + resp.StatusCode) + } + + replicas, err := st.Resources().ListByDefinition(ctx, "dst-nonodes") + if err != nil { + t.Fatalf("list the target's replicas: %v", err) + } + + if len(replicas) != 0 { + t.Errorf("the refused clone stamped %d replica(s)", len(replicas)) + } +} + +// The staleness check's volume-count arm is load-bearing in the direction the +// size arm cannot see: when the source has LOST a volume, the per-volume walk +// covers only the source's remaining ones and finds every one of them in the +// snapshot, so without the count the stale snapshot passes. +func TestRDCloneRefusesToResumeWhenTheSourceLostAVolume(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-lostvol") + + // The snapshot covers two volumes; the source now has one. + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: cloneSnapshotName("dst-lostvol"), + ResourceName: "src-lostvol", + Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{ + {VolumeNumber: 0, SizeKib: 64 * 1024}, + {VolumeNumber: 1, SizeKib: 64 * 1024}, + }, + }); err != nil { + t.Fatalf("seed the leftover snapshot: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-lostvol", map[string]any{ + "name": "dst-lostvol", + "use_zfs_clone": true, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409 — the snapshot covers a volume the source no longer has", + resp.StatusCode) + } +} + +// The third arm, and the one neither of its siblings can reach: the counts +// agree and every size that is compared agrees, because the volume the source +// now carries is not in the snapshot at all. A source that dropped one volume +// and gained another between the attempts lands exactly here — `vd d 0` then +// `vd c` renumbers rather than refills — and resuming would hydrate the +// target from a snapshot describing a volume the source no longer has, under +// a number it never had. +func TestRDCloneRefusesToResumeWhenTheSourceRenumberedItsVolume(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-renum") + + // The snapshot covers volume 0, at the size the source had it. + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: cloneSnapshotName("dst-renum"), + ResourceName: "src-renum", + Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{ + {VolumeNumber: 0, SizeKib: 64 * 1024}, + }, + }); err != nil { + t.Fatalf("seed the leftover snapshot: %v", err) + } + + // The source dropped volume 0 and took volume 1 in its place, so it + // still has exactly one volume, of exactly the captured size. + if err := st.VolumeDefinitions().Delete(ctx, "src-renum", 0); err != nil { + t.Fatalf("drop the source volume: %v", err) + } + + if err := st.VolumeDefinitions().Create(ctx, "src-renum", &apiv1.VolumeDefinition{ + VolumeNumber: 1, + SizeKib: 64 * 1024, + }); err != nil { + t.Fatalf("add the replacement volume: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-renum", map[string]any{ + "name": "dst-renum", + "use_zfs_clone": true, + }) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409 — the snapshot does not cover the volume the source now has", + resp.StatusCode) + } + + vds, err := st.VolumeDefinitions().List(ctx, "dst-renum") + if err != nil { + t.Fatalf("list the target's volumes: %v", err) + } + + if len(vds) != 0 { + t.Errorf("the refused clone hydrated %d volume(s) from the stale snapshot", len(vds)) + } +} + +// A definition stored without an explicit stack never said what its layers +// are; it does not have none. One client's attempt stamps the leftover with +// the default it resolved, another client retries without layer_list, and +// comparing the raw slices calls that a shape change — with the empty side +// rendered as nothing at all in the refusal. +func TestRDCloneResumesWhenTheLeftoverStackIsTheResolvedDefault(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-defstack") + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: cloneSnapshotName("dst-defstack"), + ResourceName: "src-defstack", + Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + }); err != nil { + t.Fatalf("seed the leftover snapshot: %v", err) + } + + // The leftover materializeRestoredRD writes off a source that records no + // stack: it records none either. + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-defstack", + Props: map[string]string{ + restoreFromSnapshotKey: restoreMarker("src-defstack", cloneSnapshotName("dst-defstack")), + }, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + // The retry names the stack linstor-csi names on every clone, which is + // what an unset stack resolves to. + resp := postClone(t, base, "src-defstack", map[string]any{ + "name": "dst-defstack", + "use_zfs_clone": true, + "layer_list": apiv1.DefaultLayerStack(), + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status = %d, want 201: an unset stack means the default, so the leftover "+ + "and the request describe the same shape", resp.StatusCode) + } + + vds, err := st.VolumeDefinitions().List(ctx, "dst-defstack") + if err != nil { + t.Fatalf("list the target's volumes: %v", err) + } + + if len(vds) != 1 { + t.Errorf("after the resume the target has %d volume(s), want 1", len(vds)) + } +} + +// blindVolumeListStore hides the target's volumes from a List while leaving +// them where a Get or a Create will find them: the pre-check's read failing, +// and the window between the two calls, produce the same shape. +type blindVolumeListStore struct { + store.Store +} + +func (b blindVolumeListStore) VolumeDefinitions() store.VolumeDefinitionStore { + return blindVolumeList{b.Store.VolumeDefinitions()} +} + +type blindVolumeList struct { + store.VolumeDefinitionStore +} + +func (b blindVolumeList) List(context.Context, string) ([]apiv1.VolumeDefinition, error) { + return nil, nil +} + +// A replay of a finished clone still has to apply the request's property +// edits. The first attempt can fail in applyClonePropEdits AFTER the volumes +// are already there, so the retry arrives at a target that looks finished with +// the edits never applied — and answering 201 then drops them, which is the +// accept-and-drop this endpoint refuses external_name and volume_passphrases +// to avoid. +func TestRDCloneReplayStillAppliesPropEdits(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-props") + + base, stop := startServerWithStore(t, st) + defer stop() + + // A finished clone, made without any edits. + first := postClone(t, base, "src-props", map[string]any{ + "name": "dst-props", + "use_zfs_clone": true, + }) + _ = first.Body.Close() + + if first.StatusCode != http.StatusCreated { + t.Fatalf("first clone: got %d, want 201", first.StatusCode) + } + + // The replay carries edits the first attempt never applied. + replay := postClone(t, base, "src-props", map[string]any{ + "name": "dst-props", + "use_zfs_clone": true, + "override_props": map[string]string{"Aux/replay": "landed"}, + }) + _ = replay.Body.Close() + + if replay.StatusCode != http.StatusCreated { + t.Fatalf("replay: got %d, want 201", replay.StatusCode) + } + + got, err := st.ResourceDefinitions().Get(ctx, "dst-props") + if err != nil { + t.Fatalf("get the clone: %v", err) + } + + if got.Props["Aux/replay"] != "landed" { + t.Errorf("the replay reported success and dropped its override_props: %v", got.Props) + } +} + +// materializeRestoredRD creates the definition, hydrates the volumes and THEN +// stamps the replicas. A leftover read on volumes alone is called finished the +// instant hydrate returns — the Bug 354 empty shell, certified complete, with +// the clone-status poll agreeing because it compares volume counts. An attempt +// that dies in that window left a target every later retry answered 201 over. +func TestRDCloneResumesALeftoverThatWasHydratedButNeverPlaced(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-unplaced") + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: cloneSnapshotName("dst-unplaced"), + ResourceName: "src-unplaced", + Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + }); err != nil { + t.Fatalf("seed the leftover snapshot: %v", err) + } + + seedCloneLeftover(t, st, "src-unplaced", "dst-unplaced") + + // Hydrated, never placed: exactly where an attempt dies between the two. + if err := st.VolumeDefinitions().Create(ctx, "dst-unplaced", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the hydrated volume: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-unplaced", map[string]any{ + "name": "dst-unplaced", + "use_zfs_clone": true, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status = %d, want 201", resp.StatusCode) + } + + replicas, err := st.Resources().ListByDefinition(ctx, "dst-unplaced") + if err != nil { + t.Fatalf("list the target's replicas: %v", err) + } + + if len(replicas) == 0 { + t.Error("the retry reported the clone complete and placed nothing") + } +} + +// A replay of a finished clone is a copy of a point-in-time and owes the source +// nothing. The shape comparison derives what it compares against from the LIVE +// source, so an ordinary `rd modify --resource-group` on the source turned +// every later replay into a 409 — with a correction linstor-csi cannot follow, +// since it sends the same body every time. +func TestRDCloneReplayOfAFinishedCloneSurvivesASourceGroupMove(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-rgmove") + + base, stop := startServerWithStore(t, st) + defer stop() + + first := postClone(t, base, "src-rgmove", map[string]any{ + "name": "dst-rgmove", + "use_zfs_clone": true, + }) + _ = first.Body.Close() + + if first.StatusCode != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", first.StatusCode) + } + + // The operator moves the SOURCE to another group. Nothing about the + // finished clone changed. + src, err := st.ResourceDefinitions().Get(ctx, "src-rgmove") + if err != nil { + t.Fatalf("read the source: %v", err) + } + + src.ResourceGroupName = "grp-moved" + if err := st.ResourceDefinitions().Update(ctx, &src); err != nil { + t.Fatalf("move the source: %v", err) + } + + replay := postClone(t, base, "src-rgmove", map[string]any{ + "name": "dst-rgmove", + "use_zfs_clone": true, + }) + _ = replay.Body.Close() + + if replay.StatusCode != http.StatusCreated { + t.Fatalf("replay = %d, want 201 — the clone was finished before the source moved", + replay.StatusCode) + } +} + +// The internal snapshot is deletable: handleSnapshotDelete has no guard for one +// a clone depends on, and operators do remove stray clone-* snapshots because +// they block deleting the source. Asking the finished question after taking one +// meant a pure replay re-snapshotted the LIVE source, and once the source had +// grown that fresh snapshot diverged from the finished target and refused the +// replay from then on — with a correction that destroys a clone holding data. +func TestRDCloneReplayWithTheInternalSnapshotGoneTakesNoNewOne(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-nosnap") + + base, stop := startServerWithStore(t, st) + defer stop() + + first := postClone(t, base, "src-nosnap", map[string]any{ + "name": "dst-nosnap", + "use_zfs_clone": true, + }) + _ = first.Body.Close() + + if first.StatusCode != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", first.StatusCode) + } + + if err := st.Snapshots().Delete(ctx, "src-nosnap", cloneSnapshotName("dst-nosnap")); err != nil { + t.Fatalf("delete the internal snapshot: %v", err) + } + + // The source grows, which is what used to poison the replay permanently. + vd, err := st.VolumeDefinitions().Get(ctx, "src-nosnap", 0) + if err != nil { + t.Fatalf("read the source volume: %v", err) + } + + vd.SizeKib = 128 * 1024 + if err := st.VolumeDefinitions().Update(ctx, "src-nosnap", &vd); err != nil { + t.Fatalf("grow the source: %v", err) + } + + for attempt := 1; attempt <= 2; attempt++ { + replay := postClone(t, base, "src-nosnap", map[string]any{ + "name": "dst-nosnap", + "use_zfs_clone": true, + }) + _ = replay.Body.Close() + + if replay.StatusCode != http.StatusCreated { + t.Fatalf("replay %d = %d, want 201", attempt, replay.StatusCode) + } + } + + // And the replay took no snapshot of the live source: a pure replay must + // not write, least of all an object claiming by its name to be this + // clone's origin while recording a different point-in-time. + if _, err := st.Snapshots().Get(ctx, "src-nosnap", cloneSnapshotName("dst-nosnap")); err == nil { + t.Error("the replay re-snapshotted the live source") + } +} + +// hydrateVolumesFromSnapshot tolerates a volume already present at the matching +// size precisely so a partial hydration can be finished, and the restore +// endpoint sharing this data plane does resume the identical state. Comparing +// volume COUNTS classified a strict prefix as somebody else's definition, so a +// multi-volume clone whose first attempt died between two creates was stranded. +func TestRDCloneResumesAHalfHydratedMultiVolumeLeftover(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-mv") + + if err := st.VolumeDefinitions().Create(ctx, "src-mv", + &apiv1.VolumeDefinition{VolumeNumber: 1, SizeKib: 32 * 1024}); err != nil { + t.Fatalf("give the source a second volume: %v", err) + } + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: cloneSnapshotName("dst-mv"), + ResourceName: "src-mv", + Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{ + {VolumeNumber: 0, SizeKib: 64 * 1024}, + {VolumeNumber: 1, SizeKib: 32 * 1024}, + }, + }); err != nil { + t.Fatalf("seed the leftover snapshot: %v", err) + } + + seedCloneLeftover(t, st, "src-mv", "dst-mv") + + // One of the two volumes landed before the first attempt died. + if err := st.VolumeDefinitions().Create(ctx, "dst-mv", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the half hydration: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-mv", map[string]any{ + "name": "dst-mv", + "use_zfs_clone": true, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status = %d, want 201 — the leftover is a prefix of what this clone restores", + resp.StatusCode) + } + + vds, err := st.VolumeDefinitions().List(ctx, "dst-mv") + if err != nil { + t.Fatalf("list the target's volumes: %v", err) + } + + if len(vds) != 2 { + t.Errorf("target has %d volume(s) after the resume, want 2", len(vds)) + } +} + +// One cluster state, two answers: the create branch refuses a clone whose +// snapshot would have to be taken on an offline node, and the reuse branch ran +// none of that. The snapshot's nodes are where the restore places the clone's +// replicas, so an offline one is the same problem whichever attempt took the +// snapshot — and the retry stamped a replica on a node whose satellite cannot +// act on it. +func TestRDCloneRefusesToResumeOntoAnOfflineNode(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-offline") + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: cloneSnapshotName("dst-offline"), + ResourceName: "src-offline", + Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + }); err != nil { + t.Fatalf("seed the leftover snapshot: %v", err) + } + + if err := st.Nodes().SetConnectionStatus(ctx, "node-a", apiv1.NodeTypeOffline); err != nil { + t.Fatalf("take the node offline: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-offline", map[string]any{ + "name": "dst-offline", + "use_zfs_clone": true, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503 — the snapshot's node is offline, the same answer "+ + "the create branch gives", resp.StatusCode) + } +} diff --git a/pkg/rest/rd_clone_recorded_volumes_test.go b/pkg/rest/rd_clone_recorded_volumes_test.go new file mode 100644 index 00000000..564d5a27 --- /dev/null +++ b/pkg/rest/rd_clone_recorded_volumes_test.go @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "encoding/json" + "net/http" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// seedTwoVolumeCloneLeftover leaves the state a clone of a two-volume source +// can be in once its internal snapshot is gone: the marker and the record of +// what the snapshot held, the given volumes, and a live replica. +func seedTwoVolumeCloneLeftover(t *testing.T, st store.Store, src, dst string, hydrated ...int32) { + t.Helper() + + ctx := t.Context() + seedDeployedCloneSource(t, st, src) + + if err := st.VolumeDefinitions().Create(ctx, src, + &apiv1.VolumeDefinition{VolumeNumber: 1, SizeKib: 32 * 1024}); err != nil { + t.Fatalf("seed the source's second volume: %v", err) + } + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: dst, + Props: map[string]string{ + restoreFromSnapshotKey: restoreMarker(src, cloneSnapshotName(dst)), + store.RestoreVolumesProp: store.EncodeRestoreVolumes([]apiv1.SnapshotVolumeDef{ + {VolumeNumber: 0, SizeKib: 64 * 1024}, + {VolumeNumber: 1, SizeKib: 32 * 1024}, + }), + }, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + sizes := map[int32]int64{0: 64 * 1024, 1: 32 * 1024} + + for _, number := range hydrated { + if err := st.VolumeDefinitions().Create(ctx, dst, + &apiv1.VolumeDefinition{VolumeNumber: number, SizeKib: sizes[number]}); err != nil { + t.Fatalf("seed volume %d of the leftover: %v", number, err) + } + } + + if err := st.Resources().Create(ctx, &apiv1.Resource{ + Name: dst, NodeName: "node-a", Props: map[string]string{"StorPoolName": "zfs-thin"}, + }); err != nil { + t.Fatalf("seed the leftover's replica: %v", err) + } +} + +func cloneStatusOf(t *testing.T, base, src, dst string) (int, string) { + t.Helper() + + resp := httpGet(t, base+"/v1/resource-definitions/"+src+"/clone/"+dst) + defer func() { _ = resp.Body.Close() }() + + var got struct { + Status string `json:"status"` + } + + _ = json.NewDecoder(resp.Body).Decode(&got) + + return resp.StatusCode, got.Status +} + +// With the clone's internal snapshot gone, a leftover holding one volume and a +// live replica read as finished whatever the snapshot held, so a two-volume +// clone that had restored only its first volume was reported complete. +func TestRDCloneJudgesALeftoverAgainstItsRecordWhenTheSnapshotIsGone(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedTwoVolumeCloneLeftover(t, st, "src-partial", "dst-partial", 0) + + base, stop := startServerWithStore(t, st) + defer stop() + + if code, status := cloneStatusOf(t, base, "src-partial", "dst-partial"); status == "COMPLETE" { + t.Errorf("clone status over a partial clone = %d %q, want not COMPLETE", code, status) + } + + if code := cloneOnce(t, base, "src-partial", "dst-partial", nil); code == http.StatusCreated { + t.Errorf("retry over a partial clone whose snapshot is gone = 201") + } +} + +// The record is what makes a whole clone stay whole once its snapshot is +// reaped: every recorded volume present and a live replica is finished. +func TestRDCloneStaysCompleteWhenItsSnapshotIsGone(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedTwoVolumeCloneLeftover(t, st, "src-whole", "dst-whole", 0, 1) + + base, stop := startServerWithStore(t, st) + defer stop() + + if code, status := cloneStatusOf(t, base, "src-whole", "dst-whole"); status != "COMPLETE" { + t.Errorf("clone status over a whole clone = %d %q, want COMPLETE", code, status) + } + + if code := cloneOnce(t, base, "src-whole", "dst-whole", nil); code != http.StatusCreated { + t.Errorf("retry over a whole clone whose snapshot is gone = %d, want 201", code) + } +} + +// A resume finishes the leftover from the snapshot it takes now, which can +// hold other volumes than the one the leftover was created from. The record +// kept from the first attempt judged the finished clone against volumes it was +// never restored with once the snapshot was gone, and the poll and a retry +// both called a working clone unfinished. +func TestRDCloneResumeRecordsTheVolumesOfTheSnapshotItTook(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-retake") + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-retake", + Props: map[string]string{ + restoreFromSnapshotKey: restoreMarker("src-retake", cloneSnapshotName("dst-retake")), + store.RestoreVolumesProp: store.EncodeRestoreVolumes([]apiv1.SnapshotVolumeDef{ + {VolumeNumber: 0, SizeKib: 64 * 1024}, + {VolumeNumber: 1, SizeKib: 32 * 1024}, + }), + }, + }); err != nil { + t.Fatalf("seed the empty leftover of a two-volume snapshot: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-retake", "dst-retake", nil); code != http.StatusCreated { + t.Fatalf("resume from a retaken snapshot = %d, want 201", code) + } + + if err := st.Snapshots().Delete(ctx, "src-retake", cloneSnapshotName("dst-retake")); err != nil { + t.Fatalf("delete the clone's snapshot: %v", err) + } + + if code, status := cloneStatusOf(t, base, "src-retake", "dst-retake"); status != "COMPLETE" { + t.Errorf("clone status once its retaken snapshot is gone = %d %q, want COMPLETE", code, status) + } + + if code := cloneOnce(t, base, "src-retake", "dst-retake", nil); code != http.StatusCreated { + t.Errorf("retry over the finished clone once its snapshot is gone = %d, want 201", code) + } +} + +// The replay of a finished clone takes the adoption mark before it answers +// 201, as the restore's replay does, so a creator rolling the clone back +// yields instead of deleting what the replay was answered for. +func TestRDCloneReplayMarksTheCloneAdopted(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedTwoVolumeCloneLeftover(t, st, "src-mark", "dst-mark", 0, 1) + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-mark", "dst-mark", nil); code != http.StatusCreated { + t.Fatalf("replay of the finished clone = %d, want 201", code) + } + + rd, err := st.ResourceDefinitions().Get(t.Context(), "dst-mark") + if err != nil { + t.Fatalf("read the clone: %v", err) + } + + if rd.Props[store.RestoreAdoptedProp] == "" { + t.Error("the replay answered 201 without marking the clone adopted") + } +} diff --git a/pkg/rest/rd_clone_replay_test.go b/pkg/rest/rd_clone_replay_test.go new file mode 100644 index 00000000..87bfe37c --- /dev/null +++ b/pkg/rest/rd_clone_replay_test.go @@ -0,0 +1,898 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "context" + "encoding/json" + "net/http" + "slices" + "strings" + "sync/atomic" + "testing" + + "github.com/cockroachdb/errors" + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// seedTwoNodeCloneSource is seedDeployedCloneSource with a second diskful +// replica, so a clone places two and "half-placed" is a state that can exist. +func seedTwoNodeCloneSource(t *testing.T, st store.Store, rdName string) { + t.Helper() + + ctx := t.Context() + seedDeployedCloneSource(t, st, rdName) + + if err := st.Nodes().Create(ctx, &apiv1.Node{Name: "node-b", ConnectionStatus: "ONLINE"}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + + if err := st.StoragePools().Create(ctx, &apiv1.StoragePool{ + StoragePoolName: "zfs-thin", + NodeName: "node-b", + ProviderKind: "ZFS_THIN", + SupportsSnapshot: true, + }); err != nil { + t.Fatalf("seed pool on node-b: %v", err) + } + + if err := st.Resources().Create(ctx, &apiv1.Resource{ + Name: rdName, + NodeName: "node-b", + Props: map[string]string{"StorPoolName": "zfs-thin"}, + }); err != nil { + t.Fatalf("seed replica on node-b: %v", err) + } +} + +func cloneOnce(t *testing.T, base, src, dst string, extra map[string]any) int { + t.Helper() + + body := map[string]any{"name": dst, "use_zfs_clone": true} + for k, v := range extra { + body[k] = v + } + + resp := postClone(t, base, src, body) + _ = resp.Body.Close() + + return resp.StatusCode +} + +// Hoisting the finished question above everything that reads the live source +// also hoisted it above everything that reads the REQUEST. A replay naming a +// shape the finished clone does not have was told the clone completed and +// handed the other shape: a caller who names LUKS gets plaintext. +func TestRDCloneReplayRefusesANamedShapeTheFinishedCloneDoesNotHave(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-shape") + + if err := st.ResourceGroups().Create(ctx, &apiv1.ResourceGroup{Name: "grp-other"}); err != nil { + t.Fatalf("seed RG: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-shape", "dst-shape", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + if code := cloneOnce(t, base, "src-shape", "dst-shape", + map[string]any{"resource_group": "grp-other"}); code == http.StatusCreated { + t.Error("replay naming a different resource_group was answered 201 with it dropped") + } + + if code := cloneOnce(t, base, "src-shape", "dst-shape", + map[string]any{"layer_list": []string{"storage"}}); code == http.StatusCreated { + t.Error("replay naming a different layer_list was answered 201 with it dropped") + } + + // The CSI replay names neither field and has to stay an idempotent success. + if code := cloneOnce(t, base, "src-shape", "dst-shape", nil); code != http.StatusCreated { + t.Errorf("plain replay = %d, want 201", code) + } +} + +// Scaling a clone down is an ordinary operation, and a replay after it must not +// undo it. Judging the clone by whether every snapshot node still carried a +// replica read the scaled-down clone as unfinished, and the resume re-stamped +// the removed replica from the point-in-time while the survivor had moved on. +// Once the source had also grown, the resume refused the replay permanently, +// with a correction that deletes a clone holding data. +func TestRDCloneReplayLeavesAScaledDownCloneAlone(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedTwoNodeCloneSource(t, st, "src-half") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-half", "dst-half", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + placed, err := st.Resources().ListByDefinition(ctx, "dst-half") + if err != nil { + t.Fatalf("list the clone's replicas: %v", err) + } + + if len(placed) != 2 { + t.Fatalf("a complete clone of a two-node source placed %d replica(s), want 2", len(placed)) + } + + if err := st.Resources().Delete(ctx, "dst-half", "node-b"); err != nil { + t.Fatalf("scale the clone down: %v", err) + } + + growSourceVolume(t, st, "src-half", 128*1024) + + if code := cloneOnce(t, base, "src-half", "dst-half", nil); code != http.StatusCreated { + t.Fatalf("replay after scaling the clone down and growing the source = %d, want 201", code) + } + + after, err := st.Resources().ListByDefinition(ctx, "dst-half") + if err != nil { + t.Fatalf("list the clone's replicas after the replay: %v", err) + } + + if len(after) != 1 || after[0].NodeName != "node-a" { + t.Errorf("replay left replicas %v, want only node-a: it re-stamped the one removed", nodeNamesOf(after)) + } +} + +func nodeNamesOf(replicas []apiv1.Resource) []string { + names := make([]string, 0, len(replicas)) + for i := range replicas { + names = append(names, replicas[i].NodeName) + } + + return names +} + +var errSnapshotReadFailed = errors.New("read the clone snapshot failed") + +type failingSnapshotGets struct{ store.SnapshotStore } + +func (failingSnapshotGets) Get(context.Context, string, string) (apiv1.Snapshot, error) { + return apiv1.Snapshot{}, errSnapshotReadFailed +} + +type failingSnapshotGetStore struct{ store.Store } + +func (f failingSnapshotGetStore) Snapshots() store.SnapshotStore { + return failingSnapshotGets{f.Store.Snapshots()} +} + +// "The snapshot does not exist" is a fact about the world; "the snapshot could +// not be read" is a fact about this request. Only the first may decide a clone +// at face value, and the replay used to take either. +func TestRDCloneReplayDoesNotTakeAnUnreadableSnapshotAtFaceValue(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-unread") + + base, stop := startServerWithStore(t, backend) + + if code := cloneOnce(t, base, "src-unread", "dst-unread", nil); code != http.StatusCreated { + stop() + t.Fatalf("first clone = %d, want 201", code) + } + + stop() + + base2, stop2 := startServerWithStore(t, failingSnapshotGetStore{backend}) + defer stop2() + + if code := cloneOnce(t, base2, "src-unread", "dst-unread", nil); code == http.StatusCreated { + t.Error("replay answered 201 on a snapshot read that failed rather than one that was absent") + } +} + +// The hoist took the live-source comparison off the replay and left it on the +// resume, which is where a retry of an unfinished clone lives. A first attempt +// that dies before placing, a source moved to another group for unrelated +// reasons, and the retry arrives with the same body it always sends. +func TestRDCloneResumeOfAnUnfinishedLeftoverSurvivesASourceGroupMove(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-unf") + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: cloneSnapshotName("dst-unf"), + ResourceName: "src-unf", + Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + }); err != nil { + t.Fatalf("seed the leftover snapshot: %v", err) + } + + seedCloneLeftover(t, st, "src-unf", "dst-unf") + + src, err := st.ResourceDefinitions().Get(ctx, "src-unf") + if err != nil { + t.Fatalf("read the source: %v", err) + } + + src.ResourceGroupName = "grp-new" + if err := st.ResourceDefinitions().Update(ctx, &src); err != nil { + t.Fatalf("move the source: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-unf", "dst-unf", nil); code != http.StatusCreated { + t.Fatalf("retry of an unfinished clone after the source moved = %d, want 201", code) + } +} + +// The POST was made independent of the live source; the GET linstor-csi polls +// right after it was not. A volume added to the source after the clone +// finished turned the poll that follows a 201 into FAILED, one request apart. +func TestRDCloneStatusPollAgreesWithTheReplayAfterTheSourceGainsAVolume(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-poll") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-poll", "dst-poll", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + if err := st.VolumeDefinitions().Create(ctx, "src-poll", + &apiv1.VolumeDefinition{VolumeNumber: 1, SizeKib: 32 * 1024}); err != nil { + t.Fatalf("give the source a second volume: %v", err) + } + + if code := cloneOnce(t, base, "src-poll", "dst-poll", nil); code != http.StatusCreated { + t.Fatalf("replay = %d, want 201", code) + } + + resp := httpGet(t, base+"/v1/resource-definitions/src-poll/clone/dst-poll") + defer func() { _ = resp.Body.Close() }() + + var got struct { + Status string `json:"status"` + } + + if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + t.Fatalf("decode clone-status: %v", err) + } + + if got.Status != "COMPLETE" { + t.Errorf("poll after a 201 replay = %q, want COMPLETE", got.Status) + } +} + +// Following the printed correction has to work. Deleting only the snapshot +// when the leftover already holds volumes hydrated from it leads through a bare +// 500 with no correction, over a fresh snapshot of the live source. +func TestRDCloneStaleSnapshotCorrectionNamesTheTargetWhenItHoldsVolumes(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-stale") + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: cloneSnapshotName("dst-stale"), + ResourceName: "src-stale", + Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + }); err != nil { + t.Fatalf("seed the leftover snapshot: %v", err) + } + + seedCloneLeftover(t, st, "src-stale", "dst-stale") + + if err := st.VolumeDefinitions().Create(ctx, "dst-stale", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the hydrated volume: %v", err) + } + + growSourceVolume(t, st, "src-stale", 128*1024) + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-stale", map[string]any{"name": "dst-stale", "use_zfs_clone": true}) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409", resp.StatusCode) + } + + var envelope cloneStartedResponse + if err := json.NewDecoder(resp.Body).Decode(&envelope); err != nil { + t.Fatalf("decode the envelope: %v", err) + } + + if envelope.Messages == nil || len(*envelope.Messages) == 0 { + t.Fatal("empty envelope") + } + + if correc := (*envelope.Messages)[0].Correc; !strings.Contains(correc, "delete 'dst-stale'") { + t.Errorf("correction %q does not name the target, so following it leads to a 500", correc) + } +} + +// A finished clone owes the snapshot nothing either. An ordinary expansion of +// the clone read as "holds a volume this clone would not have written", and the +// correction said to delete a clone with data on it. +func TestRDCloneReplayOfAnExpandedCloneIsStillFinished(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-exp") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-exp", "dst-exp", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + vd, err := st.VolumeDefinitions().Get(ctx, "dst-exp", 0) + if err != nil { + t.Fatalf("read the clone's volume: %v", err) + } + + vd.SizeKib *= 2 + if err := st.VolumeDefinitions().Update(ctx, "dst-exp", &vd); err != nil { + t.Fatalf("expand the clone: %v", err) + } + + if code := cloneOnce(t, base, "src-exp", "dst-exp", nil); code != http.StatusCreated { + t.Errorf("replay after expanding the clone = %d, want 201", code) + } +} + +// A finished clone whose definition carries the DELETE flag is being torn down +// whatever its replicas say, and the POST refuses it as that. The poll judged +// it by its replicas alone and read COMPLETE, which linstor-csi binds on +// without a POST: deleted directly rather than through `rd d`, which cascades +// the replicas first, the definition still lists an unflagged one. +func TestRDCloneStatusTreatsADeleteFlaggedDefinitionAsTornDown(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-flagged") + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-flagged", + Flags: []string{rdFlagDelete}, + Props: map[string]string{store.RestoreFromSnapshotProp: "src-flagged:" + cloneSnapshotName("dst-flagged")}, + }); err != nil { + t.Fatalf("seed the target: %v", err) + } + + if err := st.VolumeDefinitions().Create(ctx, "dst-flagged", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the target's volume: %v", err) + } + + if err := st.Resources().Create(ctx, &apiv1.Resource{Name: "dst-flagged", NodeName: "node-a"}); err != nil { + t.Fatalf("seed the target's replica: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + code, status := cloneStatusOnce(t, base+"/v1/resource-definitions/src-flagged/clone/dst-flagged") + if code != http.StatusNotFound || status == "COMPLETE" { + t.Errorf("poll of a DELETE-flagged finished clone = %d %q, want 404", code, status) + } + + if post := cloneOnce(t, base, "src-flagged", "dst-flagged", nil); post == http.StatusCreated { + t.Errorf("POST of the same clone = %d, want a refusal", post) + } +} + +// A replay of a finished clone owes the source nothing, and was still asked +// behind a read of it: a source deleted once its clone was done answered the +// replay 404, and one whose volumes were removed sent it down the empty-shell +// path to an already-exists. +func TestRDCloneReplayOfAFinishedCloneDoesNotReadTheSource(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + mutate func(t *testing.T, st store.Store, src string) + }{ + {name: "source-deleted", mutate: func(t *testing.T, st store.Store, src string) { + t.Helper() + + if err := st.ResourceDefinitions().Delete(t.Context(), src); err != nil { + t.Fatalf("delete the source: %v", err) + } + }}, + {name: "source-volumes-removed", mutate: func(t *testing.T, st store.Store, src string) { + t.Helper() + + if err := st.VolumeDefinitions().Delete(t.Context(), src, 0); err != nil { + t.Fatalf("remove the source's volume: %v", err) + } + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-gone") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-gone", "dst-gone", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + tc.mutate(t, st, "src-gone") + + if code := cloneOnce(t, base, "src-gone", "dst-gone", nil); code != http.StatusCreated { + t.Errorf("replay of the finished clone = %d, want 201", code) + } + }) + } +} + +// The poll answered a finished clone COMPLETE without the gate the POST's +// replay asks before its 201: a clone the RG-deleted rollback was deleting, or +// one whose rollback gave up, read done to the driver, which binds on the poll +// and never reaches the POST that refuses it. +func TestRDCloneStatusAsksTheGateTheReplayAsks(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + mutate func(rd *apiv1.ResourceDefinition) + }{ + {name: "rollback-in-progress", mutate: func(rd *apiv1.ResourceDefinition) { + rd.Props[store.RollbackAbandonedProp] = store.RollbackInProgress + }}, + {name: "rollback-gave-up", mutate: func(rd *apiv1.ResourceDefinition) { + rd.Props[store.RollbackAbandonedProp] = "delete-replicas" + }}, + {name: "group-gone", mutate: func(rd *apiv1.ResourceDefinition) { + rd.ResourceGroupName = "grp-gone" + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-gate") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-gate", "dst-gate", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + if err := st.ResourceDefinitions().PatchResourceDefinitionSpec(ctx, "dst-gate", + func(rd *apiv1.ResourceDefinition) error { + if rd.Props == nil { + rd.Props = map[string]string{} + } + + tc.mutate(rd) + + return nil + }); err != nil { + t.Fatalf("put the clone in the state under test: %v", err) + } + + if code, status := cloneStatusOnce(t, base+"/v1/resource-definitions/src-gate/clone/dst-gate"); status == "COMPLETE" { + t.Errorf("poll = %d %q, want the clone sent back to the POST", code, status) + } + + if code := cloneOnce(t, base, "src-gate", "dst-gate", nil); code == http.StatusCreated { + t.Errorf("POST = %d, want a refusal", code) + } + }) + } +} + +// cacheMissesTheDeleteFlag serves one definition from a cache that has not +// seen its DELETE flag; the API server holds it. +type cacheMissesTheDeleteFlag struct { + store.Store + + target string +} + +type cacheMissesTheDeleteFlagRDs struct { + store.ResourceDefinitionStore + + target string +} + +func (c cacheMissesTheDeleteFlag) ResourceDefinitions() store.ResourceDefinitionStore { + return cacheMissesTheDeleteFlagRDs{ResourceDefinitionStore: c.Store.ResourceDefinitions(), target: c.target} +} + +func (c cacheMissesTheDeleteFlagRDs) Get(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + rd, err := c.ResourceDefinitionStore.Get(ctx, name) + if err == nil && name == c.target { + rd.Flags = slices.DeleteFunc(slices.Clone(rd.Flags), func(f string) bool { return f == rdFlagDelete }) + } + + return rd, err //nolint:wrapcheck // pass-through test double +} + +// A definition deleted directly, read from a cache that has not seen its +// DELETE flag yet, was answered COMPLETE by the poll and 201 by the replay. +// The gate both share reads the flag past the cache. +func TestRDCloneAnswersADeleteTheCacheHasNotSeen(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, backend, "src-lagdel") + + plain, stopPlain := startServerWithStore(t, backend) + + if code := cloneOnce(t, plain, "src-lagdel", "dst-lagdel", nil); code != http.StatusCreated { + stopPlain() + t.Fatalf("first clone = %d, want 201", code) + } + + stopPlain() + + if err := backend.ResourceDefinitions().PatchResourceDefinitionSpec(ctx, "dst-lagdel", + func(rd *apiv1.ResourceDefinition) error { + rd.Flags = append(rd.Flags, rdFlagDelete) + + return nil + }); err != nil { + t.Fatalf("flag the clone for deletion: %v", err) + } + + base, stop := startServerWithStore(t, cacheMissesTheDeleteFlag{Store: backend, target: "dst-lagdel"}) + defer stop() + + if _, status := cloneStatusOnce(t, base+"/v1/resource-definitions/src-lagdel/clone/dst-lagdel"); status == "COMPLETE" { + t.Error("poll answered COMPLETE over a definition being deleted") + } + + if code := cloneOnce(t, base, "src-lagdel", "dst-lagdel", nil); code == http.StatusCreated { + t.Error("replay answered 201 over a definition being deleted") + } +} + +// staleSourceVolumes serves one definition's volumes from a cache that has +// not seen a resize; the API server holds the new size. +type staleSourceVolumes struct { + store.Store + + source string + stale []apiv1.VolumeDefinition +} + +type staleSourceVolumesVDs struct { + store.VolumeDefinitionStore + + outer staleSourceVolumes +} + +func (s staleSourceVolumes) VolumeDefinitions() store.VolumeDefinitionStore { + return staleSourceVolumesVDs{VolumeDefinitionStore: s.Store.VolumeDefinitions(), outer: s} +} + +func (v staleSourceVolumesVDs) List(ctx context.Context, rdName string) ([]apiv1.VolumeDefinition, error) { + if rdName == v.outer.source { + return v.outer.stale, nil + } + + return v.VolumeDefinitionStore.List(ctx, rdName) //nolint:wrapcheck // pass-through test double +} + +func (v staleSourceVolumesVDs) ListUncached(ctx context.Context, rdName string) ([]apiv1.VolumeDefinition, error) { + return v.VolumeDefinitionStore.List(ctx, rdName) //nolint:wrapcheck // the API server's answer +} + +// Whether a reused clone snapshot still describes the source is asked of the +// API server: a cache that had not seen the source's resize let a snapshot of +// the old size through as current. +func TestRDCloneSnapshotIsCurrentReadsTheSourcePastTheCache(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-cur") + + stale, err := st.VolumeDefinitions().List(ctx, "src-cur") + if err != nil { + t.Fatalf("read the source's volumes: %v", err) + } + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: cloneSnapshotName("dst-cur"), + ResourceName: "src-cur", + Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: stale[0].SizeKib}}, + }); err != nil { + t.Fatalf("seed the leftover snapshot: %v", err) + } + + seedMarkedCloneLeftover(t, st, "src-cur", "dst-cur") + growSourceVolume(t, st, "src-cur", 2*stale[0].SizeKib) + + base, stop := startServerWithStore(t, staleSourceVolumes{Store: st, source: "src-cur", stale: stale}) + defer stop() + + if code := cloneOnce(t, base, "src-cur", "dst-cur", nil); code != http.StatusConflict { + t.Errorf("resume over a snapshot the source outgrew, read through a stale cache = %d, want 409", code) + } +} + +// caseFoldingDefinitions finds a definition under any spelling of its name, +// as the Kubernetes store does, while replicas stay selected by the name +// exactly as stored. +type caseFoldingDefinitions struct{ store.Store } + +type caseFoldingDefinitionsRDs struct{ store.ResourceDefinitionStore } + +func (c caseFoldingDefinitions) ResourceDefinitions() store.ResourceDefinitionStore { + return caseFoldingDefinitionsRDs{c.Store.ResourceDefinitions()} +} + +func (r caseFoldingDefinitionsRDs) Get(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + rd, err := r.ResourceDefinitionStore.Get(ctx, name) + if errors.Is(err, store.ErrNotFound) { + return r.ResourceDefinitionStore.Get(ctx, strings.ToLower(name)) //nolint:wrapcheck // pass-through test double + } + + return rd, err //nolint:wrapcheck // pass-through test double +} + +func (r caseFoldingDefinitionsRDs) GetUncached(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + rd, err := r.ResourceDefinitionStore.GetUncached(ctx, name) + if errors.Is(err, store.ErrNotFound) { + return r.ResourceDefinitionStore.GetUncached(ctx, strings.ToLower(name)) //nolint:wrapcheck // pass-through test double + } + + return rd, err //nolint:wrapcheck // pass-through test double +} + +// A replay spelling the target in another case finds the definition, whose +// replicas are selected by the name as stored. Judged under the request's +// spelling it counted none, read as unfinished, and re-stamped a replica on +// the node the operator had emptied. It is judged under the stored name. +func TestRDCloneReplayInAnotherCaseIsJudgedUnderTheStoredName(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedTwoNodeCloneSource(t, backend, "src-case") + + base, stop := startServerWithStore(t, caseFoldingDefinitions{backend}) + defer stop() + + if code := cloneOnce(t, base, "src-case", "dst-case", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + if err := backend.Resources().Delete(ctx, "dst-case", "node-b"); err != nil { + t.Fatalf("empty node-b: %v", err) + } + + if code := cloneOnce(t, base, "src-case", "DST-CASE", nil); code != http.StatusCreated { + t.Errorf("replay in another case = %d, want 201", code) + } + + all, err := backend.Resources().List(ctx) + if err != nil { + t.Fatalf("list replicas: %v", err) + } + + var placed []string + + for i := range all { + if strings.EqualFold(all[i].Name, "dst-case") { + placed = append(placed, all[i].Name+"@"+all[i].NodeName) + } + } + + if len(placed) != 1 { + t.Errorf("the replay left replicas %v, want only dst-case@node-a", placed) + } +} + +// The restore door's twin: a placed restore replayed in another case is judged +// under the stored name and leaves the emptied node alone. +func TestSnapshotRestoreReplayInAnotherCaseIsJudgedUnderTheStoredName(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedTwoNodeSource(t, backend, "src-rcase") + seedRestoreSnapshot(t, backend, "src-rcase", "snap-rcase", []string{"node-a", "node-b"}) + + base, stop := startServerWithStore(t, caseFoldingDefinitions{backend}) + defer stop() + + body := func(name string) map[string]any { + return map[string]any{"to_resource": name, "nodes": []string{"node-a", "node-b"}} + } + + if code, rc := restoreAnswer(t, base, "src-rcase", "snap-rcase", body("dst-rcase")); code != http.StatusCreated { + t.Fatalf("first restore = %d %q, want 201", code, rc.Message) + } + + if err := backend.Resources().Delete(ctx, "dst-rcase", "node-b"); err != nil { + t.Fatalf("empty node-b: %v", err) + } + + if code, rc := restoreAnswer(t, base, "src-rcase", "snap-rcase", body("DST-RCASE")); code != http.StatusCreated { + t.Errorf("replay in another case = %d %q, want 201", code, rc.Message) + } + + all, err := backend.Resources().List(ctx) + if err != nil { + t.Fatalf("list replicas: %v", err) + } + + var placed []string + + for i := range all { + if strings.EqualFold(all[i].Name, "dst-rcase") { + placed = append(placed, all[i].Name+"@"+all[i].NodeName) + } + } + + if len(placed) != 1 { + t.Errorf("the replay left replicas %v, want only dst-rcase@node-a", placed) + } +} + +// adoptRacesTheRebind folds case the way the Kubernetes store does, Create +// included, except that the first cached read of a spelling other than the +// stored one misses: the read that names the target as it is stored lost a +// race with the request that created it, so the create collides and adopts. +type adoptRacesTheRebind struct { + store.Store + + misses *atomic.Int32 +} + +type adoptRacesTheRebindRDs struct { + store.ResourceDefinitionStore + + misses *atomic.Int32 +} + +func (a adoptRacesTheRebind) ResourceDefinitions() store.ResourceDefinitionStore { + return adoptRacesTheRebindRDs{ResourceDefinitionStore: a.Store.ResourceDefinitions(), misses: a.misses} +} + +func (r adoptRacesTheRebindRDs) Get(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + if name != strings.ToLower(name) && r.misses.Add(-1) >= 0 { + return apiv1.ResourceDefinition{}, store.ErrNotFound + } + + return r.ResourceDefinitionStore.Get(ctx, strings.ToLower(name)) //nolint:wrapcheck // pass-through test double +} + +func (r adoptRacesTheRebindRDs) GetUncached(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + return r.ResourceDefinitionStore.GetUncached(ctx, strings.ToLower(name)) //nolint:wrapcheck // pass-through test double +} + +func (r adoptRacesTheRebindRDs) Create(ctx context.Context, rd *apiv1.ResourceDefinition) error { + if _, err := r.ResourceDefinitionStore.GetUncached(ctx, strings.ToLower(rd.Name)); err == nil { + return store.ErrAlreadyExists + } + + return r.ResourceDefinitionStore.Create(ctx, rd) //nolint:wrapcheck // pass-through test double +} + +// A definition adopted through a create that collided is hydrated and placed +// under the name it is stored with: a replica stamped under the request's +// spelling is invisible to the definition's own cascade. +func TestAnAdoptedDefinitionIsPlacedUnderItsStoredName(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedTwoNodeSource(t, backend, "src-adopt") + seedRestoreSnapshot(t, backend, "src-adopt", "snap-adopt", []string{"node-a", "node-b"}) + + plain, stopPlain := startServerWithStore(t, backend) + + code, rc := restoreAnswer(t, plain, "src-adopt", "snap-adopt", + map[string]any{"to_resource": "dst-adopt", "nodes": []string{"node-a"}}) + stopPlain() + + if code != http.StatusCreated { + t.Fatalf("first restore = %d %q, want 201", code, rc.Message) + } + + misses := &atomic.Int32{} + misses.Store(1) + + base, stop := startServerWithStore(t, adoptRacesTheRebind{Store: backend, misses: misses}) + defer stop() + + if code, rc := restoreAnswer(t, base, "src-adopt", "snap-adopt", + map[string]any{"to_resource": "DST-ADOPT", "nodes": []string{"node-a", "node-b"}}); code != http.StatusCreated { + t.Fatalf("restore in another case over the leftover = %d %q, want 201", code, rc.Message) + } + + all, err := backend.Resources().List(ctx) + if err != nil { + t.Fatalf("list replicas: %v", err) + } + + for i := range all { + if all[i].Name == "DST-ADOPT" { + t.Errorf("replica on %s stamped under the request's spelling", all[i].NodeName) + } + } +} + +// The clone door's twin of TestAnAdoptedDefinitionIsPlacedUnderItsStoredName: +// a clone that adopts its leftover through a collided create goes on under +// the name the definition is stored with, so its prop edits land on it rather +// than failing over a name nothing is stored under. +func TestAnAdoptedCloneIsFinishedUnderItsStoredName(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedTwoNodeCloneSource(t, backend, "src-cadopt") + + plain, stopPlain := startServerWithStore(t, backend) + + if code := cloneOnce(t, plain, "src-cadopt", "dst-cadopt", nil); code != http.StatusCreated { + stopPlain() + t.Fatalf("first clone = %d, want 201", code) + } + + stopPlain() + + if err := backend.Resources().Delete(ctx, "dst-cadopt", "node-b"); err != nil { + t.Fatalf("leave the clone unfinished: %v", err) + } + + if err := backend.Resources().Delete(ctx, "dst-cadopt", "node-a"); err != nil { + t.Fatalf("leave the clone unfinished: %v", err) + } + + misses := &atomic.Int32{} + misses.Store(1) + + base, stop := startServerWithStore(t, adoptRacesTheRebind{Store: backend, misses: misses}) + defer stop() + + if code := cloneOnce(t, base, "src-cadopt", "DST-CADOPT", + map[string]any{"override_props": map[string]string{"Aux/edit": "x"}}); code != http.StatusCreated { + t.Fatalf("clone in another case over the leftover = %d, want 201", code) + } + + rd, err := backend.ResourceDefinitions().Get(ctx, "dst-cadopt") + if err != nil { + t.Fatalf("read the clone: %v", err) + } + + if rd.Props["Aux/edit"] != "x" { + t.Errorf("the clone's prop edit did not land on the stored definition: %v", rd.Props) + } +} diff --git a/pkg/rest/rd_clone_resume_point_in_time_test.go b/pkg/rest/rd_clone_resume_point_in_time_test.go new file mode 100644 index 00000000..ece5becb --- /dev/null +++ b/pkg/rest/rd_clone_resume_point_in_time_test.go @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "context" + "net/http" + "strings" + "sync/atomic" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// snapshotGoneAfterFirstRead answers the first read of one snapshot from the +// backend and deletes it on every later one: the clone's internal snapshot +// reaped between the leftover judgement and the resume taking a snapshot. +type snapshotGoneAfterFirstRead struct { + store.Store + + source, snapshot string + reads *atomic.Int32 +} + +type snapshotGoneAfterFirstReadSnaps struct { + store.SnapshotStore + + outer snapshotGoneAfterFirstRead +} + +func (s snapshotGoneAfterFirstRead) Snapshots() store.SnapshotStore { + return snapshotGoneAfterFirstReadSnaps{SnapshotStore: s.Store.Snapshots(), outer: s} +} + +func (s snapshotGoneAfterFirstReadSnaps) Get(ctx context.Context, rdName, snapName string) (apiv1.Snapshot, error) { + if strings.EqualFold(rdName, s.outer.source) && strings.EqualFold(snapName, s.outer.snapshot) && + s.outer.reads.Add(1) > 1 { + _ = s.Delete(ctx, rdName, snapName) + } + + return s.SnapshotStore.Get(ctx, rdName, snapName) //nolint:wrapcheck // pass-through test double +} + +// ListByDefinitionUncached counts as a read of the snapshot too: the resume +// asks the API server whether it is still there. +func (s snapshotGoneAfterFirstReadSnaps) ListByDefinitionUncached(ctx context.Context, rdName string) ([]apiv1.Snapshot, error) { + if strings.EqualFold(rdName, s.outer.source) && s.outer.reads.Add(1) > 1 { + _ = s.Delete(ctx, rdName, s.outer.snapshot) + } + + return s.SnapshotStore.ListByDefinitionUncached(ctx, rdName) //nolint:wrapcheck // pass-through test double +} + +// A leftover already restored from the clone's snapshot, a volume with a +// replica, was finished from a fresh snapshot of the source as it is now once +// the first one was gone: two moments of the source joined into one clone. +func TestRDCloneResumeRefusesToFinishFromALaterPointInTime(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, backend, "src-pit") + + if err := backend.VolumeDefinitions().Create(ctx, "src-pit", + &apiv1.VolumeDefinition{VolumeNumber: 1, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("give the source a second volume: %v", err) + } + + plain, stopPlain := startServerWithStore(t, backend) + + if code := cloneOnce(t, plain, "src-pit", "dst-pit", nil); code != http.StatusCreated { + stopPlain() + t.Fatalf("first clone = %d, want 201", code) + } + + stopPlain() + + // What a first attempt that died between two volumes leaves: one volume + // and its replica, restored from the snapshot, the second volume missing. + if err := backend.VolumeDefinitions().Delete(ctx, "dst-pit", 1); err != nil { + t.Fatalf("leave the second volume missing: %v", err) + } + + base, stop := startServerWithStore(t, snapshotGoneAfterFirstRead{ + Store: backend, source: "src-pit", snapshot: cloneSnapshotName("dst-pit"), reads: &atomic.Int32{}, + }) + defer stop() + + if code := cloneOnce(t, base, "src-pit", "dst-pit", nil); code != http.StatusConflict { + t.Errorf("resume over a restored leftover whose snapshot went = %d, want 409", code) + } + + if _, err := backend.Snapshots().Get(ctx, "src-pit", cloneSnapshotName("dst-pit")); err == nil { + t.Error("the resume took a new snapshot of the source to finish the leftover from") + } + + if vds, _ := backend.VolumeDefinitions().List(ctx, "dst-pit"); len(vds) != 1 { + t.Errorf("the refused resume wrote into the leftover: %d volume(s)", len(vds)) + } +} + +// cacheMissesTheCloneSnapshot answers every cached read of one snapshot after +// the first NotFound, as an informer that dropped it on a relist and has not +// caught up does; the API server holds it throughout. +type cacheMissesTheCloneSnapshot struct { + store.Store + + source, snapshot string + reads *atomic.Int32 +} + +type cacheMissesTheCloneSnapshotSnaps struct { + store.SnapshotStore + + outer cacheMissesTheCloneSnapshot +} + +func (c cacheMissesTheCloneSnapshot) Snapshots() store.SnapshotStore { + return cacheMissesTheCloneSnapshotSnaps{SnapshotStore: c.Store.Snapshots(), outer: c} +} + +func (c cacheMissesTheCloneSnapshotSnaps) Get(ctx context.Context, rdName, snapName string) (apiv1.Snapshot, error) { + if strings.EqualFold(rdName, c.outer.source) && strings.EqualFold(snapName, c.outer.snapshot) && + c.outer.reads.Add(1) > 1 { + return apiv1.Snapshot{}, store.ErrNotFound + } + + return c.SnapshotStore.Get(ctx, rdName, snapName) //nolint:wrapcheck // pass-through test double +} + +// The point-in-time gate refuses a clone and tells the operator to delete it, +// so it asks the API server: a cache that had not seen the snapshot refused a +// leftover whose snapshot was right there. +func TestRDCloneResumeAsksTheAPIServerWhetherItsSnapshotIsGone(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, backend, "src-pitc") + + if err := backend.VolumeDefinitions().Create(ctx, "src-pitc", + &apiv1.VolumeDefinition{VolumeNumber: 1, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("give the source a second volume: %v", err) + } + + plain, stopPlain := startServerWithStore(t, backend) + + if code := cloneOnce(t, plain, "src-pitc", "dst-pitc", nil); code != http.StatusCreated { + stopPlain() + t.Fatalf("first clone = %d, want 201", code) + } + + stopPlain() + + if err := backend.VolumeDefinitions().Delete(ctx, "dst-pitc", 1); err != nil { + t.Fatalf("leave the second volume missing: %v", err) + } + + base, stop := startServerWithStore(t, cacheMissesTheCloneSnapshot{ + Store: backend, source: "src-pitc", snapshot: cloneSnapshotName("dst-pitc"), reads: &atomic.Int32{}, + }) + defer stop() + + if code := cloneOnce(t, base, "src-pitc", "dst-pitc", nil); code == http.StatusConflict { + t.Error("resume over a leftover whose snapshot the cache had not seen = 409") + } +} + +// cacheKeepsTheCloneSnapshot answers cached reads of one snapshot with a copy +// taken before it was deleted, as an informer that has not seen the delete +// yet does; the API server no longer holds it. +type cacheKeepsTheCloneSnapshot struct { + store.Store + + stale apiv1.Snapshot +} + +type cacheKeepsTheCloneSnapshotSnaps struct { + store.SnapshotStore + + outer cacheKeepsTheCloneSnapshot +} + +func (c cacheKeepsTheCloneSnapshot) Snapshots() store.SnapshotStore { + return cacheKeepsTheCloneSnapshotSnaps{SnapshotStore: c.Store.Snapshots(), outer: c} +} + +func (c cacheKeepsTheCloneSnapshotSnaps) Get(ctx context.Context, rdName, snapName string) (apiv1.Snapshot, error) { + if strings.EqualFold(rdName, c.outer.stale.ResourceName) && strings.EqualFold(snapName, c.outer.stale.Name) { + return c.outer.stale, nil + } + + return c.SnapshotStore.Get(ctx, rdName, snapName) //nolint:wrapcheck // pass-through test double +} + +// The resume asks the API server whether the clone's snapshot is gone, and +// with no replica on the leftover finishes it from a new one. The snapshot +// was then looked up again through the cache, which had not seen the delete, +// and the leftover was finished from the snapshot the first read had just +// found gone: replicas restoring from nothing. +func TestRDCloneResumeRetakesTheSnapshotACacheStillHolds(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, backend, "src-stale") + + plain, stopPlain := startServerWithStore(t, backend) + + if code := cloneOnce(t, plain, "src-stale", "dst-stale", nil); code != http.StatusCreated { + stopPlain() + t.Fatalf("first clone = %d, want 201", code) + } + + stopPlain() + + snapName := cloneSnapshotName("dst-stale") + + stale, err := backend.Snapshots().Get(ctx, "src-stale", snapName) + if err != nil { + t.Fatalf("read the clone's snapshot: %v", err) + } + + replicas, err := backend.Resources().ListByDefinition(ctx, "dst-stale") + if err != nil { + t.Fatalf("list the clone's replicas: %v", err) + } + + for i := range replicas { + if err := backend.Resources().Delete(ctx, "dst-stale", replicas[i].NodeName); err != nil { + t.Fatalf("leave the leftover without a replica: %v", err) + } + } + + if err := backend.Snapshots().Delete(ctx, "src-stale", snapName); err != nil { + t.Fatalf("delete the clone's snapshot: %v", err) + } + + base, stop := startServerWithStore(t, cacheKeepsTheCloneSnapshot{Store: backend, stale: stale}) + defer stop() + + if code := cloneOnce(t, base, "src-stale", "dst-stale", nil); code != http.StatusCreated { + t.Fatalf("resume of a leftover with no replica and its snapshot gone = %d, want 201", code) + } + + if _, err := liveCloneSnapshot(ctx, backend, "src-stale", snapName); err != nil { + t.Errorf("the resume finished the leftover from a snapshot that is gone: %v", err) + } +} + +// A leftover with no replica whose snapshot is gone is finished from a new +// snapshot like a fresh clone, unless its volumes do not fit the source as it +// is now. Smaller than the source's, the retake wrote a snapshot of a moment +// the leftover was never restored from and failed hydrating over it with a +// bare already-exists; a volume the source no longer has was kept beside the +// retake, a volume nothing restored. Both are refused first, naming the way +// out, with nothing taken. A leftover grown past the source, or missing a +// volume the source has, still fits and is finished. +func TestRDCloneResumeRefusesARetakeTheLeftoverDoesNotFit(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + leftover []apiv1.VolumeDefinition + sourceKib int64 + srcVol1 bool + fits bool + }{ + {name: "smaller", leftover: []apiv1.VolumeDefinition{{VolumeNumber: 0, SizeKib: 64 * 1024}}, sourceKib: 128 * 1024}, + { + name: "volume-the-source-lacks", + leftover: []apiv1.VolumeDefinition{{VolumeNumber: 0, SizeKib: 64 * 1024}, {VolumeNumber: 1, SizeKib: 64 * 1024}}, + }, + {name: "grown", leftover: []apiv1.VolumeDefinition{{VolumeNumber: 0, SizeKib: 128 * 1024}}, fits: true}, + { + name: "missing-a-volume", srcVol1: true, fits: true, + leftover: []apiv1.VolumeDefinition{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, backend, "src-fit") + + if tc.srcVol1 { + if err := backend.VolumeDefinitions().Create(ctx, "src-fit", + &apiv1.VolumeDefinition{VolumeNumber: 1, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("give the source a second volume: %v", err) + } + } + + if tc.sourceKib > 0 { + growSourceVolume(t, backend, "src-fit", tc.sourceKib) + } + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-fit", + Props: map[string]string{restoreFromSnapshotKey: restoreMarker("src-fit", cloneSnapshotName("dst-fit"))}, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + for i := range tc.leftover { + if err := backend.VolumeDefinitions().Create(ctx, "dst-fit", &tc.leftover[i]); err != nil { + t.Fatalf("seed the leftover's volume: %v", err) + } + } + + base, stop := startServerWithStore(t, backend) + defer stop() + + resp := postClone(t, base, "src-fit", map[string]any{"name": "dst-fit", "use_zfs_clone": true}) + rc := decodeCloneMessage(t, resp) + _ = resp.Body.Close() + + _, snapErr := backend.Snapshots().Get(ctx, "src-fit", cloneSnapshotName("dst-fit")) + + if tc.fits { + if resp.StatusCode != http.StatusCreated { + t.Errorf("resume over a leftover that fits = %d %q, want 201", resp.StatusCode, rc.Message) + } + + return + } + + if resp.StatusCode != http.StatusConflict || !strings.Contains(rc.Correc, "delete 'dst-fit' and clone again") { + t.Errorf("resume over a leftover the source no longer fits = %d %q / %q, want 409 naming the way out", + resp.StatusCode, rc.Message, rc.Correc) + } + + if snapErr == nil { + t.Error("the refused resume took a new snapshot of the source") + } + }) + } +} + +// A diskless replica holds none of the earlier moment's data, so a leftover +// whose only replica is diskless is finished from a new snapshot like one with +// no replica at all, rather than refused as holding data from a snapshot that +// is gone. +func TestRDCloneResumeRetakesOverALeftoverWithOnlyADisklessReplica(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-dlr") + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-dlr", + Props: map[string]string{restoreFromSnapshotKey: restoreMarker("src-dlr", cloneSnapshotName("dst-dlr"))}, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + if err := st.VolumeDefinitions().Create(ctx, "dst-dlr", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the leftover's volume: %v", err) + } + + if err := st.Resources().Create(ctx, &apiv1.Resource{ + Name: "dst-dlr", NodeName: "node-b", Flags: []string{apiv1.ResourceFlagDiskless}, + }); err != nil { + t.Fatalf("seed the diskless replica: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-dlr", "dst-dlr", nil); code != http.StatusCreated { + t.Errorf("resume of a leftover whose only replica is diskless = %d, want 201", code) + } +} diff --git a/pkg/rest/rd_clone_resume_test.go b/pkg/rest/rd_clone_resume_test.go new file mode 100644 index 00000000..e1a0bc4e --- /dev/null +++ b/pkg/rest/rd_clone_resume_test.go @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "net/http" + "strings" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// "Found" is not "still right". The first attempt takes the internal snapshot +// and dies; the source is resized before the retry; the retry hydrates from +// the stale snapshot and answers 201, and the clone-status poll then reports +// COMPLETE because the volume counts agree. The caller gets a clone at the old +// size with nothing saying so — reachable only since the retry started +// resuming instead of reporting the leftover done. +func TestRDCloneRefusesToResumeOverAStaleSnapshot(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-stale") + + // What the first attempt left: the snapshot, taken at the old size. + snap := apiv1.Snapshot{ + Name: cloneSnapshotName("dst-stale"), + ResourceName: "src-stale", + Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + } + if err := st.Snapshots().Create(ctx, &snap); err != nil { + t.Fatalf("seed the leftover snapshot: %v", err) + } + + // The source grew between the attempts. + vd, err := st.VolumeDefinitions().Get(ctx, "src-stale", 0) + if err != nil { + t.Fatalf("read the source volume: %v", err) + } + + vd.SizeKib = 128 * 1024 + if err := st.VolumeDefinitions().Update(ctx, "src-stale", &vd); err != nil { + t.Fatalf("grow the source: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-stale", map[string]any{ + "name": "dst-stale", + "use_zfs_clone": true, + }) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409 — the leftover snapshot is a volume size behind", + resp.StatusCode) + } + + vds, err := st.VolumeDefinitions().List(ctx, "dst-stale") + if err != nil { + t.Fatalf("list the target's volumes: %v", err) + } + + if len(vds) != 0 { + t.Errorf("the refused clone hydrated %d volume(s) from the stale snapshot", len(vds)) + } +} + +// The positive control: the same resume, over a snapshot that still describes +// the source, still completes. Without it the refusal above could be coming +// from the resume path itself. +func TestRDCloneResumesOverACurrentSnapshot(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-current") + + snap := apiv1.Snapshot{ + Name: cloneSnapshotName("dst-current"), + ResourceName: "src-current", + Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + } + if err := st.Snapshots().Create(ctx, &snap); err != nil { + t.Fatalf("seed the leftover snapshot: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-current", map[string]any{ + "name": "dst-current", + "use_zfs_clone": true, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status = %d, want 201", resp.StatusCode) + } + + vds, err := st.VolumeDefinitions().List(ctx, "dst-current") + if err != nil { + t.Fatalf("list the target's volumes: %v", err) + } + + if len(vds) != 1 { + t.Errorf("target has %d volume(s), want 1", len(vds)) + } +} + +// The clone half of the case-fold. A retry spelling the target in another case +// addresses the same object — pkg/store/k8s/crdname.go lowercases every lookup +// key — so a marker comparison built from what the caller typed answered that +// somebody else owns the leftover, refused every retry, and left the target +// with no volumes forever. This is the clone sibling of the restore-side +// case-fold test. +func TestRDCloneResumesWhateverCaseTheRetryUses(t *testing.T) { + t.Parallel() + + st := caseFoldingStore{store.NewInMemory()} + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-case") + + // What the first attempt left, stored lower-case. + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: cloneSnapshotName("dst-case"), + ResourceName: "src-case", + Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + }); err != nil { + t.Fatalf("seed the leftover snapshot: %v", err) + } + + seedCloneLeftover(t, st, "src-case", "dst-case") + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "SRC-CASE", map[string]any{ + "name": "DST-CASE", + "use_zfs_clone": true, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("retry spelled DST-CASE over a leftover stored dst-case: got %d, want 201", + resp.StatusCode) + } + + vds, err := st.VolumeDefinitions().List(ctx, "dst-case") + if err != nil { + t.Fatalf("list the target's volumes: %v", err) + } + + if len(vds) != 1 { + t.Errorf("after the retry the target has %d volume(s), want 1", len(vds)) + } + + // The retry has to have RESUMED the leftover, not created a second + // definition beside it. Everything above is satisfied either way, so + // without this the test passes over a store whose Create does not fold — + // which is what the real one does, and what the shim has to imitate for + // the case-fold to be under test at all. + rds, err := st.ResourceDefinitions().List(ctx) + if err != nil { + t.Fatalf("list definitions: %v", err) + } + + targets := 0 + + for i := range rds { + if strings.EqualFold(rds[i].Name, "dst-case") { + targets++ + } + } + + if targets != 1 { + t.Errorf("%d definitions under the target name; the retry created a second one "+ + "beside the leftover instead of resuming it", targets) + } + + // And the marker the resume wrote is the stored spelling, not the one the + // retry typed: the satellite splits this value to find the source, and a + // clone made through the CLI has to be recognisable to a REST retry and + // the other way round. + got, err := st.ResourceDefinitions().Get(ctx, "dst-case") + if err != nil { + t.Fatalf("get the target: %v", err) + } + + if want := restoreMarker("src-case", cloneSnapshotName("dst-case")); got.Props[restoreFromSnapshotKey] != want { + t.Errorf("marker = %q, want %q — written from the stored objects, not from the request", + got.Props[restoreFromSnapshotKey], want) + } +} + +// A resumed clone keeps the definition the first attempt created, so a retry +// naming a different shape gets that shape validated and then dropped while +// the answer says the clone completed. The parent group decides replica count +// and pool selection, so it is not cosmetic — and accepting a field and +// dropping it is what external_name and volume_passphrases are refused for. +func TestRDCloneRefusesARetryWithADifferentShape(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-shape") + seedCloneRG(t, st, "grp-a", "grp-b") + + // The leftover, parented to the group the first attempt asked for. + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-shape", + ResourceGroupName: "grp-a", + Props: map[string]string{ + restoreFromSnapshotKey: restoreMarker("src-shape", cloneSnapshotName("dst-shape")), + }, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-shape", map[string]any{ + "name": "dst-shape", + "resource_group": "grp-b", + "use_zfs_clone": true, + }) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409 — the retry asks for a group the leftover was not started with", + resp.StatusCode) + } + + got, err := st.ResourceDefinitions().Get(ctx, "dst-shape") + if err != nil { + t.Fatalf("get the leftover: %v", err) + } + + if got.ResourceGroupName != "grp-a" { + t.Errorf("resource group = %q; the refused retry re-parented the definition", + got.ResourceGroupName) + } +} + +// The narrow race the pre-check cannot see: another request completes the +// restore and the target is deleted between the state check and this Create. +// The AlreadyExists fallback re-makes the decision on fresh state, and it has +// to re-make all of it — hydrating volumes into a dying definition is exactly +// what the 409 above prevents. +func TestMaterializeRefusesADyingLeftoverItRacedInto(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedRestoreSource(ctx, t, st) + + snap, err := st.Snapshots().Get(ctx, "pvc-src", "snap-1") + if err != nil { + t.Fatalf("read the seeded snapshot: %v", err) + } + + // The state another request left behind mid-tear-down. + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-raced", + Flags: []string{rdFlagDelete}, + Props: map[string]string{ + restoreFromSnapshotKey: restoreMarker("pvc-src", "snap-1"), + }, + }); err != nil { + t.Fatalf("seed the dying leftover: %v", err) + } + + srv := &Server{Store: st} + req := &snapshotRestoreRequest{ToResource: "pvc-raced"} + + _, err = srv.materializeRestoredRD(ctx, "pvc-src", req, &snap, false, nil) + if err == nil { + t.Fatal("materialize accepted a target being deleted") + } + + vds, listErr := st.VolumeDefinitions().List(ctx, "pvc-raced") + if listErr != nil { + t.Fatalf("list the leftover's volumes: %v", listErr) + } + + if len(vds) != 0 { + t.Errorf("hydrated %d volume(s) into a definition being deleted", len(vds)) + } +} diff --git a/pkg/rest/rd_clone_server_owned_props_test.go b/pkg/rest/rd_clone_server_owned_props_test.go new file mode 100644 index 00000000..78e5964f --- /dev/null +++ b/pkg/rest/rd_clone_server_owned_props_test.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "errors" + "net/http" + "testing" + + "github.com/cozystack/blockstor/pkg/store" +) + +// The prop edits land on the target after the clone stamped its marker, so an +// edit that removes or rewrites it would leave the satellite to bring the +// volumes up blank. Every edit shape that reaches the marker is refused before +// anything is written. +func TestRDCloneRefusesPropEditsThatReachItsMarker(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + body map[string]any + }{ + {name: "override", body: map[string]any{ + "override_props": map[string]string{store.RestoreFromSnapshotProp: "other:snap"}, + }}, + {name: "delete", body: map[string]any{ + "delete_props": []string{store.RestoreFromSnapshotProp}, + }}, + {name: "namespace", body: map[string]any{ + "delete_namespaces": []string{"Blockstor"}, + }}, + {name: "volumes-record", body: map[string]any{ + "override_props": map[string]string{store.RestoreVolumesProp: "0:1"}, + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-owned") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-owned", "dst-owned", tc.body); code != http.StatusBadRequest { + t.Errorf("clone with an edit reaching a server-owned prop = %d, want 400", code) + } + + if _, err := st.ResourceDefinitions().Get(t.Context(), "dst-owned"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the refused clone wrote its target: %v", err) + } + + if snaps, _ := st.Snapshots().ListByDefinition(t.Context(), "src-owned"); len(snaps) > 0 { + t.Errorf("the refused clone took %d snapshot(s) on the source", len(snaps)) + } + }) + } +} + +// linstor-csi's own edit on every clone stays accepted. +func TestRDCloneAcceptsLinstorCSIsOwnPropEdit(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-csiedit") + + base, stop := startServerWithStore(t, st) + defer stop() + + code := cloneOnce(t, base, "src-csiedit", "dst-csiedit", map[string]any{ + "delete_props": []string{"Aux/csi-provisioning-completed-by"}, + }) + if code != http.StatusCreated { + t.Fatalf("clone with linstor-csi's delete_props = %d, want 201", code) + } + + rd, err := st.ResourceDefinitions().Get(t.Context(), "dst-csiedit") + if err != nil { + t.Fatalf("read the clone: %v", err) + } + + if rd.Props[store.RestoreFromSnapshotProp] == "" { + t.Error("the clone lost its restore marker") + } +} diff --git a/pkg/rest/rd_clone_target_test.go b/pkg/rest/rd_clone_target_test.go new file mode 100644 index 00000000..6332fdc4 --- /dev/null +++ b/pkg/rest/rd_clone_target_test.go @@ -0,0 +1,332 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "context" + "encoding/json" + "io" + "net/http" + "strings" + "testing" + + "github.com/cockroachdb/errors" + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// The volume-less door started from a copy of the source, flags included, and +// had no delete gate: a clone taken inside the source's delete window came back +// carrying DELETE for good. +func TestRDCloneOfAVolumelessSourceBeingDeletedIsRefused(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "src-dying", Flags: []string{rdFlagDelete}, + }); err != nil { + t.Fatalf("seed the dying source: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-dying", map[string]any{"name": "dst-dying"}) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("clone of a volume-less source being deleted = %d, want 409", resp.StatusCode) + } + + if _, err := st.ResourceDefinitions().Get(ctx, "dst-dying"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the refused clone left a target behind: %v", err) + } +} + +func TestRDCloneOfAVolumelessSourceDoesNotInheritItsFlags(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "src-flag", Flags: []string{"FAILED"}, + }); err != nil { + t.Fatalf("seed the flagged source: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-flag", map[string]any{"name": "dst-flag"}) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("clone = %d, want 201", resp.StatusCode) + } + + dst, err := st.ResourceDefinitions().Get(ctx, "dst-flag") + if err != nil { + t.Fatalf("read the clone: %v", err) + } + + if len(dst.Flags) != 0 { + t.Errorf("clone flags = %v, want none: a shell took the source's lifecycle with it", dst.Flags) + } +} + +type failingVolumeLists struct{ store.VolumeDefinitionStore } + +func (failingVolumeLists) List(context.Context, string) ([]apiv1.VolumeDefinition, error) { + return nil, errSnapshotReadFailed +} + +type failingVolumeListStore struct{ store.Store } + +func (f failingVolumeListStore) VolumeDefinitions() store.VolumeDefinitionStore { + return failingVolumeLists{f.Store.VolumeDefinitions()} +} + +// python-linstor hands every non-2xx clone body to CloneStarted and reads its +// messages, so a bare []ApiCallRc loses the operator's error to an +// AttributeError. Every refusal on the POST has to be the object. +func TestRDCloneRefusalsKeepTheCloneStartedEnvelope(t *testing.T) { + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-env") + + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{Name: "src-shell"}); err != nil { + t.Fatalf("seed the volume-less source: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + blind, stopBlind := startServerWithStore(t, failingVolumeListStore{st}) + defer stopBlind() + + if code := cloneOnce(t, base, "src-shell", "dst-shell", nil); code != http.StatusCreated { + t.Fatalf("first volume-less clone = %d, want 201", code) + } + + for _, tc := range []struct { + name, base, src string + body map[string]any + }{ + {name: "missing-source", base: base, src: "no-such-src", body: map[string]any{"name": "dst-x"}}, + {name: "no-name", base: base, src: "src-env", body: map[string]any{"use_zfs_clone": true}}, + {name: "volume-less-replay", base: base, src: "src-shell", body: map[string]any{"name": "dst-shell"}}, + {name: "source-volumes-unreadable", base: blind, src: "src-env", body: map[string]any{"name": "dst-y"}}, + } { + t.Run(tc.name, func(t *testing.T) { + resp := postClone(t, tc.base, tc.src, tc.body) + defer func() { _ = resp.Body.Close() }() + + raw, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read the body: %v", err) + } + + if resp.StatusCode < http.StatusBadRequest { + t.Fatalf("status = %d, want a refusal", resp.StatusCode) + } + + var envelope cloneStartedResponse + if err := json.Unmarshal(raw, &envelope); err != nil || envelope.Messages == nil || len(*envelope.Messages) == 0 { + t.Errorf("refusal body is not a CloneStarted object with messages: %s", raw) + } + }) + } +} + +// Only a leftover that is not being deleted may be answered as a finished +// replay. Every existing DELETE fixture seeds an unfinished leftover, so the +// term never decided anything there. +func TestRDCloneReplayOfAFinishedCloneBeingDeletedIsNotAReplay(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-fd") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-fd", "dst-fd", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + dst, err := st.ResourceDefinitions().Get(ctx, "dst-fd") + if err != nil { + t.Fatalf("read the clone: %v", err) + } + + dst.Flags = append(dst.Flags, rdFlagDelete) + if err := st.ResourceDefinitions().Update(ctx, &dst); err != nil { + t.Fatalf("mark the clone for deletion: %v", err) + } + + if code := cloneOnce(t, base, "src-fd", "dst-fd", nil); code != http.StatusConflict { + t.Errorf("replay over a finished clone being deleted = %d, want 409", code) + } +} + +// A volume the snapshot never recorded, on a target with no replica, is not +// this clone's: resuming would hydrate around it and report complete a +// definition carrying a volume nothing restored. +func TestRDCloneRefusesATargetHoldingAVolumeTheSnapshotNeverRecorded(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-ex") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-ex", "dst-ex", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + if err := st.Resources().Delete(ctx, "dst-ex", "node-a"); err != nil { + t.Fatalf("drop the replica: %v", err) + } + + if err := st.VolumeDefinitions().Create(ctx, "dst-ex", + &apiv1.VolumeDefinition{VolumeNumber: 7, SizeKib: 32 * 1024}); err != nil { + t.Fatalf("add a volume the snapshot never recorded: %v", err) + } + + if code := cloneOnce(t, base, "src-ex", "dst-ex", nil); code != http.StatusConflict { + t.Errorf("retry over a target with an unrecorded volume and no replica = %d, want 409", code) + } + + if code, _ := cloneStatusOnce(t, base+"/v1/resource-definitions/src-ex/clone/dst-ex"); code != http.StatusNotFound { + t.Errorf("poll = %d, want 404", code) + } +} + +// `vd create` on a finished clone is an ordinary operation, like expanding one. +func TestRDCloneReplayOfAFinishedCloneThatGainedAVolume(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-gv") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-gv", "dst-gv", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + if err := st.VolumeDefinitions().Create(ctx, "dst-gv", + &apiv1.VolumeDefinition{VolumeNumber: 1, SizeKib: 32 * 1024}); err != nil { + t.Fatalf("add a volume to the finished clone: %v", err) + } + + if code := cloneOnce(t, base, "src-gv", "dst-gv", nil); code != http.StatusCreated { + t.Errorf("replay of a finished clone that gained a volume = %d, want 201", code) + } + + if _, status := cloneStatusOnce(t, base+"/v1/resource-definitions/src-gv/clone/dst-gv"); status != "COMPLETE" { + t.Errorf("poll = %q, want COMPLETE", status) + } +} + +// Renumbering keeps the volume count and the size, so only this arm names what +// changed. The status alone came out the same without it. +func TestRDCloneStaleSnapshotNamesTheVolumeItDoesNotCover(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-cov") + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: cloneSnapshotName("dst-cov"), + ResourceName: "src-cov", + Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + }); err != nil { + t.Fatalf("seed the leftover snapshot: %v", err) + } + + if err := st.VolumeDefinitions().Delete(ctx, "src-cov", 0); err != nil { + t.Fatalf("drop the source volume: %v", err) + } + + if err := st.VolumeDefinitions().Create(ctx, "src-cov", + &apiv1.VolumeDefinition{VolumeNumber: 1, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("add the replacement volume: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := postClone(t, base, "src-cov", map[string]any{"name": "dst-cov", "use_zfs_clone": true}) + defer func() { _ = resp.Body.Close() }() + + envelope := decodeCloneStarted(t, resp) + if envelope.Messages == nil || len(*envelope.Messages) == 0 { + t.Fatalf("status %d with an empty envelope", resp.StatusCode) + } + + if msg := (*envelope.Messages)[0].Message; !strings.Contains(msg, "does not cover volume 1") { + t.Errorf("refusal %q does not name the volume the snapshot misses", msg) + } +} + +// Beside a snapshot volume that is still missing, a volume the snapshot never +// recorded is somebody else's even when a replica exists: the resume would +// hydrate the missing one next to it and report the clone complete. +func TestRDCloneRefusesAPartialTargetHoldingAnUnrecordedVolume(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-mx") + + if err := st.VolumeDefinitions().Create(ctx, "src-mx", + &apiv1.VolumeDefinition{VolumeNumber: 1, SizeKib: 32 * 1024}); err != nil { + t.Fatalf("give the source a second volume: %v", err) + } + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: cloneSnapshotName("dst-mx"), + ResourceName: "src-mx", + Nodes: []string{"node-a"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{ + {VolumeNumber: 0, SizeKib: 64 * 1024}, + {VolumeNumber: 1, SizeKib: 32 * 1024}, + }, + }); err != nil { + t.Fatalf("seed the snapshot: %v", err) + } + + seedCloneLeftover(t, st, "src-mx", "dst-mx") + + for _, vd := range []apiv1.VolumeDefinition{ + {VolumeNumber: 0, SizeKib: 64 * 1024}, + {VolumeNumber: 7, SizeKib: 8 * 1024}, + } { + if err := st.VolumeDefinitions().Create(ctx, "dst-mx", &vd); err != nil { + t.Fatalf("seed leftover volume %d: %v", vd.VolumeNumber, err) + } + } + + if err := st.Resources().Create(ctx, &apiv1.Resource{ + Name: "dst-mx", NodeName: "node-a", Props: map[string]string{"StorPoolName": "zfs-thin"}, + }); err != nil { + t.Fatalf("seed the leftover replica: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-mx", "dst-mx", nil); code != http.StatusConflict { + t.Errorf("retry over a partial target holding an unrecorded volume = %d, want 409", code) + } +} diff --git a/pkg/rest/rd_clone_validation_test.go b/pkg/rest/rd_clone_validation_test.go new file mode 100644 index 00000000..f13e1d81 --- /dev/null +++ b/pkg/rest/rd_clone_validation_test.go @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "encoding/json" + "io" + "net/http" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + blockstoriov1alpha1 "github.com/cozystack/blockstor/api/v1alpha1" + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// Every other door that creates a definition validates the name it is handed. +// The clone door checked only that it was non-empty, so it could create a +// definition `rd create` answers 400 for. +func TestRDCloneRefusesANameRDCreateWouldRefuse(t *testing.T) { + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-name") + + base, stop := startServerWithStore(t, st) + defer stop() + + for _, tc := range []struct{ name, target string }{ + {name: "dots", target: "dst.name.8"}, + {name: "leading-digit", target: "8dst"}, + {name: "space", target: "dst name"}, + {name: "past-the-ceiling-once-prefixed", target: strings.Repeat("d", 48)}, + } { + t.Run(tc.name, func(t *testing.T) { + code := cloneOnce(t, base, "src-name", tc.target, nil) + if code != http.StatusBadRequest { + t.Errorf("clone into %q = %d, want 400", tc.target, code) + } + }) + } +} + +// The body decode is the last refusal on this endpoint that answered outside +// the envelope, and python-linstor reads `messages` off whatever comes back. +func TestRDCloneBodyDecodeRefusalsKeepTheEnvelope(t *testing.T) { + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-dec") + + base, stop := startServerWithStore(t, st) + defer stop() + + for _, tc := range []struct{ name, body string }{ + {name: "malformed", body: `{"name":`}, + {name: "unknown-field", body: `{"name":"dst-dec","no_such_field":1}`}, + {name: "wrong-type", body: `{"name":42}`}, + {name: "empty", body: ``}, + {name: "trailing-data", body: `{"name":"dst-dec"}garbage`}, + } { + t.Run(tc.name, func(t *testing.T) { + resp := httpPost(t, base+"/v1/resource-definitions/src-dec/clone", []byte(tc.body)) + defer func() { _ = resp.Body.Close() }() + + raw, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read the body: %v", err) + } + + if resp.StatusCode < http.StatusBadRequest { + t.Fatalf("status = %d, want a refusal", resp.StatusCode) + } + + var envelope cloneStartedResponse + if err := json.Unmarshal(raw, &envelope); err != nil || + envelope.Messages == nil || len(*envelope.Messages) == 0 { + t.Errorf("decode refusal is not a CloneStarted object with messages: %s", raw) + } + }) + } +} + +// A volume-less clone takes no snapshot, so the derived name's ceiling does not +// apply to it, and a name `rd create` accepts has to stay possible. +func TestRDCloneAppliesTheSnapshotNameCeilingOnlyWhereASnapshotIsTaken(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-ceil") + + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{Name: "shell-ceil"}); err != nil { + t.Fatalf("seed the volume-less source: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + long := "d" + strings.Repeat("x", 42) + + if code := cloneOnce(t, base, "shell-ceil", long, nil); code != http.StatusCreated { + t.Errorf("volume-less clone into a %d-char name = %d, want 201", len(long), code) + } + + if code := cloneOnce(t, base, "src-ceil", long+"y", nil); code != http.StatusBadRequest { + t.Errorf("data-path clone whose snapshot name passes the ceiling = %d, want 400", code) + } +} + +// Only a missing passphrase is the caller's to fix. A Secret that could not be +// read was answered 400, which linstor-csi takes as permanent. +func TestRDCloneAnswersAnUnreadableLUKSPrerequisiteAsAServerError(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedCloneSourceWithStack(t, st, "src-luks", []string{"DRBD", "LUKS", "STORAGE"}) + + base, stop := startServerCustom(t, &Server{ + Addr: pickFreeAddr(t), + Store: st, + Client: secretsForbiddenClient(t), + Namespace: testRESTNamespace, + }) + defer stop() + + code := cloneOnce(t, base, "src-luks", "dst-luks", + map[string]any{"layer_list": []string{"DRBD", "LUKS", "STORAGE"}}) + if code != http.StatusInternalServerError { + t.Errorf("clone with an unreadable passphrase Secret = %d, want 500", code) + } +} + +// A cluster without a passphrase is the caller's to fix, and stays a 400. +func TestRDCloneAnswersAMissingLUKSPassphraseAsAClientError(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedCloneSourceWithStack(t, st, "src-nopass", []string{"DRBD", "LUKS", "STORAGE"}) + + base, stop := startServerCustom(t, &Server{ + Addr: pickFreeAddr(t), + Store: st, + Client: newFakeRESTClient(t), + Namespace: testRESTNamespace, + }) + defer stop() + + code := cloneOnce(t, base, "src-nopass", "dst-nopass", + map[string]any{"layer_list": []string{"DRBD", "LUKS", "STORAGE"}}) + if code != http.StatusBadRequest { + t.Errorf("clone on a cluster with no passphrase = %d, want 400", code) + } +} + +// A clone that names no stack takes the source's. Of a LUKS source, it needs +// the passphrase as much as one that names LUKS, and was written without it, +// for the satellite to fail on later. +func TestRDCloneOfALUKSSourceNeedsThePassphraseWithoutALayerList(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedCloneSourceWithStack(t, st, "src-inherit", []string{"DRBD", "LUKS", "STORAGE"}) + + base, stop := startServerCustom(t, &Server{ + Addr: pickFreeAddr(t), + Store: st, + Client: newFakeRESTClient(t), + Namespace: testRESTNamespace, + }) + defer stop() + + if code := cloneOnce(t, base, "src-inherit", "dst-inherit", nil); code != http.StatusBadRequest { + t.Errorf("clone of a LUKS source with no layer_list and no passphrase = %d, want 400", code) + } + + if _, err := st.ResourceDefinitions().Get(t.Context(), "dst-inherit"); err == nil { + t.Error("the refused clone wrote its target") + } +} + +// The replay of a finished LUKS clone writes no new LUKS volume, so a +// passphrase Secret that cannot be read at that moment does not fail it: a +// driver that lost the first answer would otherwise be refused a clone that +// is already done. +func TestRDCloneReplayOfAFinishedLUKSCloneDoesNotNeedThePassphrase(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedCloneSourceWithStack(t, st, "src-lreplay", []string{"DRBD", "LUKS", "STORAGE"}) + + body := map[string]any{"layer_list": []string{"DRBD", "LUKS", "STORAGE"}} + + withPass := newFakeRESTClient(t) + if err := withPass.Create(t.Context(), &blockstoriov1alpha1.ControllerConfig{ + ObjectMeta: metav1.ObjectMeta{Name: blockstoriov1alpha1.ControllerConfigName}, + Spec: blockstoriov1alpha1.ControllerConfigSpec{ + ExtraProps: map[string]string{"DrbdOptions/EncryptPassphrase": "blockstorpass"}, + }, + }); err != nil { + t.Fatalf("seed the passphrase: %v", err) + } + + plain, stopPlain := startServerCustom(t, &Server{ + Addr: pickFreeAddr(t), Store: st, Client: withPass, Namespace: testRESTNamespace, + }) + + first := postClone(t, plain, "src-lreplay", map[string]any{ + "name": "dst-lreplay", "use_zfs_clone": true, "layer_list": body["layer_list"], + }) + rc := decodeCloneMessage(t, first) + _ = first.Body.Close() + + if first.StatusCode != http.StatusCreated { + stopPlain() + t.Fatalf("first clone = %d %q, want 201", first.StatusCode, rc.Message) + } + + stopPlain() + + base, stop := startServerCustom(t, &Server{ + Addr: pickFreeAddr(t), + Store: st, + Client: secretsForbiddenClient(t), + Namespace: testRESTNamespace, + }) + defer stop() + + if code := cloneOnce(t, base, "src-lreplay", "dst-lreplay", body); code != http.StatusCreated { + t.Errorf("replay of a finished LUKS clone with the passphrase unreadable = %d, want 201", code) + } +} + +// The refusal's twin: with the passphrase set, a clone of a LUKS source that +// names no stack goes through and takes the source's stack. +func TestRDCloneOfALUKSSourceWithThePassphraseInheritsItsStack(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedCloneSourceWithStack(t, st, "src-luksok", []string{"DRBD", "LUKS", "STORAGE"}) + + base, stop := startServerWithPassphrase(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-luksok", "dst-luksok", nil); code != http.StatusCreated { + t.Fatalf("clone of a LUKS source with the passphrase set = %d, want 201", code) + } + + rd, err := st.ResourceDefinitions().Get(t.Context(), "dst-luksok") + if err != nil { + t.Fatalf("read the clone: %v", err) + } + + if !apiv1.LayerInStack(rd.LayerStack, apiv1.LayerKindLUKS) { + t.Errorf("clone stack %v, want the source's LUKS stack", rd.LayerStack) + } +} diff --git a/pkg/rest/rd_modify_server_owned_props_test.go b/pkg/rest/rd_modify_server_owned_props_test.go new file mode 100644 index 00000000..70d474eb --- /dev/null +++ b/pkg/rest/rd_modify_server_owned_props_test.go @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "encoding/json" + "net/http" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// A modify that deleted the Blockstor namespace mid-restore took the adoption +// mark with it, and the creator's rollback then reaped a definition a retry +// had already answered for. The props blockstor writes on a definition are +// refused on modify the way the clone door refuses them. +func TestRDModifyRefusesEditsOfServerOwnedProps(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + body map[string]any + }{ + {name: "override", body: map[string]any{"override_props": map[string]string{store.RestoreFromSnapshotProp: "other:snap"}}}, + {name: "delete", body: map[string]any{"delete_props": []string{store.RestoreAdoptedProp}}}, + {name: "namespace", body: map[string]any{"delete_namespaces": []string{"Blockstor"}}}, + {name: "volumes", body: map[string]any{"override_props": map[string]string{store.RestoreVolumesProp: "0:1"}}}, + {name: "set-rollback-mark", body: map[string]any{ + "override_props": map[string]string{store.RollbackAbandonedProp: store.RollbackInProgress}, + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + + props := map[string]string{ + store.RestoreFromSnapshotProp: "src:snap", + store.RestoreAdoptedProp: "2026-10-07T00:00:00Z", + "Aux/keep": "yes", + } + + if err := st.ResourceDefinitions().Create(ctx, + &apiv1.ResourceDefinition{Name: "rd-owned", Props: props}); err != nil { + t.Fatalf("seed: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + raw, err := json.Marshal(tc.body) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + resp := httpPut(t, base+"/v1/resource-definitions/rd-owned", raw) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("modify touching a server-owned prop = %d, want 400", resp.StatusCode) + } + + rd, err := st.ResourceDefinitions().Get(ctx, "rd-owned") + if err != nil { + t.Fatalf("read back: %v", err) + } + + for key, want := range props { + if rd.Props[key] != want { + t.Errorf("refused modify changed %s: %q, want %q", key, rd.Props[key], want) + } + } + }) + } +} + +// A create carrying a server-owned prop made a definition born restoring from +// another definition's snapshot, or refused by every resume over a rollback +// that never ran. It is refused the way the modify is. +func TestRDCreateRefusesServerOwnedProps(t *testing.T) { + t.Parallel() + + for _, key := range []string{ + store.RestoreFromSnapshotProp, store.RollbackAbandonedProp, + store.RestoreAdoptedProp, store.RestoreVolumesProp, + } { + t.Run(key, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + + base, stop := startServerWithStore(t, st) + defer stop() + + raw, err := json.Marshal(map[string]any{"resource_definition": map[string]any{ + "name": "rd-born", "props": map[string]string{key: "x", "Aux/keep": "yes"}, + }}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + resp := httpPost(t, base+"/v1/resource-definitions", raw) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("create carrying %s = %d, want 400", key, resp.StatusCode) + } + + if _, err := st.ResourceDefinitions().Get(t.Context(), "rd-born"); err == nil { + t.Errorf("the refused create stored the definition") + } + }) + } +} + +// The record of what the snapshot held is written by every clone, so the +// judgement that falls back on it has something to read once the snapshot +// is gone. +func TestRDCloneRecordsTheVolumesItsSnapshotHeld(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-rec") + + if err := st.VolumeDefinitions().Create(ctx, "src-rec", + &apiv1.VolumeDefinition{VolumeNumber: 1, SizeKib: 32 * 1024}); err != nil { + t.Fatalf("seed the second volume: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-rec", "dst-rec", nil); code != http.StatusCreated { + t.Fatalf("clone = %d, want 201", code) + } + + rd, err := st.ResourceDefinitions().Get(ctx, "dst-rec") + if err != nil { + t.Fatalf("read the clone: %v", err) + } + + vols, ok := store.RecordedRestoreVolumes(rd.Props) + if !ok || len(vols) != 2 { + t.Errorf("clone recorded %q, want both volumes of its snapshot", rd.Props[store.RestoreVolumesProp]) + } +} diff --git a/pkg/rest/resource_definitions.go b/pkg/rest/resource_definitions.go index 0adb8865..3e893dc4 100644 --- a/pkg/rest/resource_definitions.go +++ b/pkg/rest/resource_definitions.go @@ -20,6 +20,7 @@ package rest import ( "context" + "maps" "net/http" "github.com/cockroachdb/errors" @@ -452,6 +453,21 @@ func (s *Server) validateRDCreateBody(w http.ResponseWriter, r *http.Request, bo return false } + // The props blockstor reads back to decide what to do with a definition + // are refused on modify; a create carrying them would make a definition + // born restoring from somebody else's snapshot, or refused by every + // resume over a rollback that never ran. + if key := store.ServerOwnedPropEdit(rd.Props, nil, nil); key != "" { + writeJSON(w, http.StatusBadRequest, []apiv1.APICallRc{{ + RetCode: apiCallRcError, + Message: "resource definition '" + rd.Name + "': props carry " + key + ", which blockstor sets itself", + Cause: key + " records what blockstor did to this definition, and is read back to decide what to do next", + Correc: "drop " + key + " from the definition's props", + }}) + + return false + } + err := s.validateRDLayerStackOnCreate(r.Context(), body, rd) if err != nil { writeError(w, http.StatusBadRequest, err.Error()) @@ -483,34 +499,61 @@ func (s *Server) validateRDCreateBody(w http.ResponseWriter, r *http.Request, bo // caller may proceed, false when the HTTP error has already been // written. func (s *Server) refuseRDCreateOnUnknownRG(w http.ResponseWriter, r *http.Request, rd *apiv1.ResourceDefinition) bool { - if rd.ResourceGroupName == "" { - return true + found, err := s.lookupPinnedRG(r.Context(), rd.ResourceGroupName) + if err != nil { + writeStoreError(w, err) + + return false } - // CreateVolume hot path: linstor-csi ensures the StorageClass's RG - // (POST /v1/resource-groups) and POSTs the RD referencing it - // back-to-back; the local informer cache may not have observed the - // RG write yet. Retry the NotFound under the standard budget so - // the Bug 134 gate doesn't refuse a perfectly valid create — see - // pkg/rest/cache_retry.go. A real typo still 404s after the budget. - _, err := getRGWithCacheRetry(r.Context(), s.Store, rd.ResourceGroupName) + if !found { + writeError(w, http.StatusNotFound, unknownRGMessage(rd.ResourceGroupName)) + + return false + } + + return true +} + +// lookupPinnedRG resolves a resource group a caller pinned by name. found is +// false with a nil error when the group simply is not there; a non-nil error +// is a store failure, which is a different answer and deserves a different +// response. +// +// CreateVolume hot path: linstor-csi ensures the StorageClass's RG (POST +// /v1/resource-groups) and POSTs the RD referencing it back-to-back; the +// local informer cache may not have observed the RG write yet. Retrying the +// NotFound under the standard budget keeps the Bug 134 gate from refusing a +// perfectly valid create — see pkg/rest/cache_retry.go. A real typo still +// comes back unknown once the budget is spent. +// +// The empty name means the caller pinned no group, which is not an error on +// any of the endpoints that ask. +func (s *Server) lookupPinnedRG(ctx context.Context, rgName string) (bool, error) { + if rgName == "" { + return true, nil + } + + _, err := getRGWithCacheRetry(ctx, s.Store, rgName) if err == nil { - return true + return true, nil } if errors.Is(err, store.ErrNotFound) { - writeError(w, http.StatusNotFound, - "resource group '"+rd.ResourceGroupName+ - "' not found: create the resource group first with "+ - "`linstor rg c ` or pass a valid existing "+ - "resource group name") - - return false + return false, nil } - writeStoreError(w, err) + return false, err +} - return false +// unknownRGMessage is the single wording for a pinned group that is not +// there, so an operator reads the same correction whichever endpoint refused +// them for it. +func unknownRGMessage(rgName string) string { + return "resource group '" + rgName + + "' not found: create the resource group first with " + + "`linstor rg c ` or pass a valid existing " + + "resource group name" } // refuseRDUpdateOnUnknownRG is Bug 372's gate, the symmetric pair to @@ -679,6 +722,10 @@ func seedAutoQuorumDefaults(rd *apiv1.ResourceDefinition) { } } +// layerListField is the wire name of the layer stack, used where the value is +// reported back to the caller rather than decoded. +const layerListField = "layer_list" + // mergeRDCreateLayerInputs reconciles the three wire shapes // `POST /v1/resource-definitions` accepts for the layer // composition: @@ -710,7 +757,7 @@ func mergeRDCreateLayerInputs(body *apiv1.ResourceDefinitionCreate, rd *apiv1.Re fromLayerData := layerStackFromLayerData(rd.LayerData) views := []rdLayerInputView{ - {label: "layer_list", stack: body.LayerList}, + {label: layerListField, stack: body.LayerList}, {label: "resource_definition.layer_stack", stack: rd.LayerStack}, {label: "resource_definition.layer_data", stack: fromLayerData}, } @@ -1056,24 +1103,27 @@ func (s *Server) handleRDUpdate(w http.ResponseWriter, r *http.Request) { return } - err := s.Store.ResourceDefinitions().PatchResourceDefinitionSpec(r.Context(), name, func(rd *apiv1.ResourceDefinition) error { - // Corner-case B5 / I1: shared property-modify core — an empty - // override value deletes the key (upstream `set-property KEY ""` - // = delete-property), matching the RG path and the UG9 NOTE. - rd.Props = applyPropsModify(rd.Props, patch.OverrideProps, patch.DeleteProps) + // The props blockstor writes to record what a definition is, and reads + // back to decide what to do with it, are not the caller's to edit: a + // delete_namespaces of "Blockstor" mid-restore took the adoption mark + // with it, and a creator's rollback then reaped a definition a retry had + // already answered for. The clone door refuses the same edits. + if refuseServerOwnedModify(w, name, &patch) { + return + } - // The body declares delete_namespaces and the merge dropped it, so a - // modify carrying it answered 200 and changed nothing. Declaring two - // thirds of a props-modify envelope and silently discarding the rest - // tells the operator work happened when it did not. - deletePropNamespaces(rd.Props, patch.DeleteNamespaces) + err := s.Store.ResourceDefinitions().PatchResourceDefinitionSpec(r.Context(), name, + func(rd *apiv1.ResourceDefinition) error { return applyRDModify(rd, &patch, rgChange) }) + if errors.Is(err, store.ErrRollbackStepMarkKept) { + writeJSON(w, http.StatusBadRequest, []apiv1.APICallRc{{ + RetCode: apiCallRcError, + Message: "resource definition '" + name + "': " + store.ErrRollbackStepMarkKept.Error(), + Correc: "delete '" + name + "' by hand", + }}) - if rgChange != "" { - rd.ResourceGroupName = rgChange - } + return + } - return nil - }) if err != nil { writeStoreError(w, err) @@ -1086,6 +1136,56 @@ func (s *Server) handleRDUpdate(w http.ResponseWriter, r *http.Request) { }}) } +// refuseServerOwnedModify answers an `rd modify` that would edit a prop +// blockstor sets itself; see store.ServerOwnedPropEditByOperator. True means +// the refusal has been written. +func refuseServerOwnedModify(w http.ResponseWriter, name string, patch *resourceDefinitionModifyBody) bool { + key := store.ServerOwnedPropEditByOperator(patch.OverrideProps, patch.DeleteProps, patch.DeleteNamespaces) + if key == "" { + return false + } + + writeJSON(w, http.StatusBadRequest, []apiv1.APICallRc{{ + RetCode: apiCallRcError, + Message: "resource definition '" + name + "': the prop edits would change " + key + + ", which blockstor sets itself", + Cause: key + " records what blockstor did to this definition, and is read back to decide what to do next", + Correc: "drop " + key + " from override_props, delete_props and delete_namespaces", + }}) + + return true +} + +// applyRDModify applies an `rd modify` body to the definition inside its +// patch. +func applyRDModify(rd *apiv1.ResourceDefinition, patch *resourceDefinitionModifyBody, rgChange string) error { + // Corner-case B5 / I1: shared property-modify core — an empty + // override value deletes the key (upstream `set-property KEY ""` + // = delete-property), matching the RG path and the UG9 NOTE. + before := maps.Clone(rd.Props) + + rd.Props = applyPropsModify(rd.Props, patch.OverrideProps, patch.DeleteProps) + + // The body declares delete_namespaces, so it is honoured rather than + // accepted and dropped: answering 200 for an edit that did not happen + // is the reason the clone paths refuse what they cannot honour. + deletePropNamespaces(rd.Props, patch.DeleteNamespaces) + + // Judged against the mark the definition holds now, inside the patch: + // only a mark over a definition the rollback left whole is the + // operator's to clear; see store.RollbackMarkClearRefusal. + err := store.RollbackMarkClearRefusal(before, rd.Props) + if err != nil { + return err //nolint:wrapcheck // answered as a 400 by the caller + } + + if rgChange != "" { + rd.ResourceGroupName = rgChange + } + + return nil +} + // handleRDDelete drops an RD plus cascades the per-replica teardown. // // Idempotent on NotFound (Bug 56 sibling): CSI spec § DeleteVolume diff --git a/pkg/rest/resource_group_extras.go b/pkg/rest/resource_group_extras.go index a00e75c5..d6878a69 100644 --- a/pkg/rest/resource_group_extras.go +++ b/pkg/rest/resource_group_extras.go @@ -277,15 +277,31 @@ func (s *Server) handleVGUpdate(w http.ResponseWriter, r *http.Request) { return } + // golinstor's VolumeGroupModify, whole: a field left undeclared is a 400 + // from the strict decoder to any caller that sets it. var in struct { - OverrideProps map[string]string `json:"override_props,omitempty"` - DeleteProps []string `json:"delete_props,omitempty"` + OverrideProps map[string]string `json:"override_props,omitempty"` + Flags []string `json:"flags,omitempty"` + DeleteProps []string `json:"delete_props,omitempty"` + DeleteNamespaces []string `json:"delete_namespaces,omitempty"` } if !decodeJSON(w, r, &in) { return } + // The one volume-group flag upstream has is GROSS_SIZE, and nothing here + // sizes a spawned volume gross, so taking a flag would answer 200 for a + // request that changes nothing. Refused before any write, so a body that + // also edits props does not land half of itself. + if len(in.Flags) > 0 { + writeError(w, http.StatusNotImplemented, + "volume-group flags are not implemented: spawned volumes are sized net, so "+ + "GROSS_SIZE would change nothing; see docs/cli-parity-known-deltas.md") + + return + } + // Bug 204b shape: typed-Patch with retry-on-conflict — the // vlmNr lookup and the prop merge re-run against the live RG on // every retry, so a concurrent reconciler / sibling write can't @@ -295,6 +311,7 @@ func (s *Server) handleVGUpdate(w http.ResponseWriter, r *http.Request) { for i := range rg.VolumeGroups { if rg.VolumeGroups[i].VolumeNumber == vlmNr { mergeVGProps(&rg.VolumeGroups[i], in.OverrideProps, in.DeleteProps) + deletePropNamespaces(rg.VolumeGroups[i].Props, in.DeleteNamespaces) return nil } diff --git a/pkg/rest/resource_groups.go b/pkg/rest/resource_groups.go index 56d4b173..b4f28240 100644 --- a/pkg/rest/resource_groups.go +++ b/pkg/rest/resource_groups.go @@ -170,6 +170,10 @@ func (s *Server) handleRGCreate(w http.ResponseWriter, r *http.Request) { return } + if refuseServerOwnedGroupProps(w, rg.Name, rg.Props, rg.OverrideProps) { + return + } + // A group's stack is inherited by every definition spawned from it, so // a LUKS layer here reaches more volumes than one on a single // definition. handleRDCreate has refused this since Bug 95; leaving the @@ -232,24 +236,10 @@ func (s *Server) handleRGUpdate(w http.ResponseWriter, r *http.Request) { // it to PatchResourceGroup. See rgUpdateWireGate — keeps the // wire-validation rules out of the store error path and the handler // under the funlen budget. - gateErr := rgUpdateWireGate(raw, &patch) - if gateErr != nil { - writeError(w, http.StatusBadRequest, gateErr.Error()) - + if s.refuseRGUpdate(w, r, name, raw, &patch) { return } - // Guarding create alone leaves the stack patchable afterwards, which is - // the same door by another name. - if patch.SelectFilter.LayerStack != nil { - luksErr := s.refuseLUKSWithoutPassphrase(r.Context(), patch.SelectFilter.LayerStack) - if luksErr != nil { - writeError(w, http.StatusBadRequest, luksErr.Error()) - - return - } - } - err := s.Store.ResourceGroups().PatchResourceGroup(r.Context(), name, func(existing *apiv1.ResourceGroup) error { prevFilter := existing.SelectFilter applyRGUpdatePatch(existing, &patch, raw) @@ -624,6 +614,79 @@ func rgSelectFilterClearTable() map[string]func(*apiv1.AutoSelectFilter) { } } +// refuseRGUpdate runs every refusal an `rg modify` body meets before it is +// merged. True means the refusal has been written. +func (s *Server) refuseRGUpdate( + w http.ResponseWriter, r *http.Request, name string, raw []byte, patch *apiv1.ResourceGroup, +) bool { + gateErr := rgUpdateWireGate(raw, patch) + if gateErr != nil { + writeError(w, http.StatusBadRequest, gateErr.Error()) + + return true + } + + if refuseServerOwnedGroupProps(w, name, patch.Props, patch.OverrideProps) { + return true + } + + // Guarding create alone leaves the stack patchable afterwards, which is + // the same door by another name. + if patch.SelectFilter.LayerStack != nil { + luksErr := s.refuseLUKSWithoutPassphrase(r.Context(), patch.SelectFilter.LayerStack) + if luksErr != nil { + writeError(w, http.StatusBadRequest, luksErr.Error()) + + return true + } + } + + return false +} + +// refuseServerOwnedGroupProps answers a group create or modify that would set +// one of the props blockstor reads back off a definition. Spawn no longer +// copies them, but a group is where linstor-csi lands StorageClass +// parameters, so refusing them here keeps the group from holding a value +// that means something on a definition. Removing one stays allowed: that is +// how an operator cleans a group written before this check, so an override to +// an empty value, the upstream spelling of a delete (`set-property KEY ""`), +// is let through with delete_props and delete_namespaces. True means the +// refusal has been written. +func refuseServerOwnedGroupProps(w http.ResponseWriter, name string, props, overrideProps map[string]string) bool { + key := store.ServerOwnedPropEdit(props, nil, nil) + if key == "" { + key = store.ServerOwnedPropEdit(settingOverrides(overrideProps), nil, nil) + } + + if key == "" { + return false + } + + writeJSON(w, http.StatusBadRequest, []apiv1.APICallRc{{ + RetCode: apiCallRcError, + Message: "resource group '" + name + "': props carry " + key + ", which blockstor sets itself", + Cause: key + " records what blockstor did to a resource definition, and is read back to decide what to do next", + Correc: "drop " + key + " from the group's props", + }}) + + return true +} + +// settingOverrides drops the overrides that delete a key, an empty value, +// leaving the ones that set one. +func settingOverrides(overrideProps map[string]string) map[string]string { + setting := make(map[string]string, len(overrideProps)) + + for key, value := range overrideProps { + if value != "" { + setting[key] = value + } + } + + return setting +} + // mergeRGProps applies the OverrideProps / DeleteProps merge // semantic LINSTOR uses for any property-bag-bearing object: // override entries land first, then delete entries strip their keys. diff --git a/pkg/rest/resource_modify.go b/pkg/rest/resource_modify.go index d781db7e..d9003ddc 100644 --- a/pkg/rest/resource_modify.go +++ b/pkg/rest/resource_modify.go @@ -84,6 +84,9 @@ func (s *Server) handleResourceModify(w http.ResponseWriter, r *http.Request) { // handleControllerPropsModify / handleNodeUpdate behaviour. // I1: empty override value deletes the key (set-property KEY ""). applyPropsModify(res.Props, patch.OverrideProps, patch.DeleteProps) + // The body declares delete_namespaces; merging only the first two + // halves of the envelope answered 200 and changed nothing. + deletePropNamespaces(res.Props, patch.DeleteNamespace) return nil }) diff --git a/pkg/rest/restore_effective_layers_test.go b/pkg/rest/restore_effective_layers_test.go new file mode 100644 index 00000000..cdf9470f --- /dev/null +++ b/pkg/rest/restore_effective_layers_test.go @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "net/http" + "slices" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +func seedLayeredGroup(t *testing.T, st store.Store, name string, stack []string) { + t.Helper() + + if err := st.ResourceGroups().Create(t.Context(), &apiv1.ResourceGroup{ + Name: name, SelectFilter: apiv1.AutoSelectFilter{LayerStack: stack}, + }); err != nil { + t.Fatalf("seed group %q: %v", name, err) + } +} + +func assertDataPlaneStack(t *testing.T, st store.Store, rdName string) { + t.Helper() + + rd, err := st.ResourceDefinitions().Get(t.Context(), rdName) + if err != nil { + t.Fatalf("read %q: %v", rdName, err) + } + + if !slices.Equal(rd.LayerStack, apiv1.DefaultLayerStack()) { + t.Errorf("%q stack = %v, want the source's %v stamped: left empty, the control plane "+ + "would judge it by its group's stack while the satellite brings it up as the source's", + rdName, rd.LayerStack, apiv1.DefaultLayerStack()) + } +} + +// A clone named into a group whose stack differs keeps the stack the source's +// bytes were written under, stamped, so the group's stack cannot reach it. +func TestRDCloneIntoAGroupOfAnotherStackKeepsTheSourcesStack(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-grp") + seedLayeredGroup(t, st, "grp-luks", []string{"DRBD", "LUKS", "STORAGE"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-grp", "dst-grp", map[string]any{"resource_group": "grp-luks"}); code != http.StatusCreated { + t.Fatalf("clone into a group of another stack = %d, want 201", code) + } + + assertDataPlaneStack(t, st, "dst-grp") +} + +// A volume-less source is cloned as a shell, and the shell is stamped with the +// source's data-plane stack the same way the data path stamps its target: left +// empty under a group of another stack, volumes added to it later would be +// brought up as the default while the control plane judged them by the group. +func TestRDCloneShellIntoAGroupOfAnotherStackKeepsTheSourcesStack(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedLayeredGroup(t, st, "grp-shell-luks", []string{"DRBD", "LUKS", "STORAGE"}) + + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{Name: "src-shell"}); err != nil { + t.Fatalf("seed the volume-less source: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-shell", "dst-shell", + map[string]any{"resource_group": "grp-shell-luks"}); code != http.StatusCreated { + t.Fatalf("shell clone into a group of another stack = %d, want 201", code) + } + + assertDataPlaneStack(t, st, "dst-shell") +} + +// The shell clone stores the stack the caller named in canonical case too. +func TestRDCloneShellStoresALowercaseLayerListCanonically(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{Name: "src-case"}); err != nil { + t.Fatalf("seed the volume-less source: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-case", "dst-case", + map[string]any{"layer_list": []string{"drbd", "storage"}}); code != http.StatusCreated { + t.Fatalf("shell clone with a lowercase layer_list = %d, want 201", code) + } + + dst, err := st.ResourceDefinitions().Get(t.Context(), "dst-case") + if err != nil { + t.Fatalf("read the clone: %v", err) + } + + if !slices.Equal(dst.LayerStack, []string{"DRBD", "STORAGE"}) { + t.Errorf("layer stack = %v, want [DRBD STORAGE]", dst.LayerStack) + } +} + +// A restore inherits the source's group, whose stack may since differ from +// the one the source was created with. +func TestSnapshotRestoreStampsTheSourcesDataPlaneStack(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-rst") + seedLayeredGroup(t, st, "grp-changed", []string{"DRBD", "LUKS", "STORAGE"}) + + src, err := st.ResourceDefinitions().Get(ctx, "src-rst") + if err != nil { + t.Fatalf("read the source: %v", err) + } + + src.ResourceGroupName = "grp-changed" + if err := st.ResourceDefinitions().Update(ctx, &src); err != nil { + t.Fatalf("move the source under the changed group: %v", err) + } + + seedRestoreSnapshot(t, st, "src-rst", "snap-rst", []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := restoreOnce(t, base, "src-rst", "snap-rst", map[string]any{"to_resource": "dst-rst"}); code != http.StatusCreated { + t.Fatalf("restore = %d, want 201", code) + } + + assertDataPlaneStack(t, st, "dst-rst") +} + +// A prepared target stored without a stack is brought up with the default, +// whatever its group says, and is judged and stamped by that. +func TestSnapshotRestoreIntoAPreparedTargetStampsItsDataPlaneStack(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-prep") + seedRestoreSnapshot(t, st, "src-prep", "snap-prep", []string{"node-a"}) + seedLayeredGroup(t, st, "grp-sc", []string{"DRBD", "LUKS", "STORAGE"}) + + snap, err := st.Snapshots().Get(ctx, "src-prep", "snap-prep") + if err != nil { + t.Fatalf("read the snapshot: %v", err) + } + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-prep", ResourceGroupName: "grp-sc", + }); err != nil { + t.Fatalf("seed the prepared target: %v", err) + } + + for _, svd := range snap.VolumeDefinitions { + if err := st.VolumeDefinitions().Create(ctx, "pvc-prep", + &apiv1.VolumeDefinition{VolumeNumber: svd.VolumeNumber, SizeKib: svd.SizeKib}); err != nil { + t.Fatalf("seed the prepared volume: %v", err) + } + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := restoreOnce(t, base, "src-prep", "snap-prep", map[string]any{ + "to_resource": "pvc-prep", "nodes": []string{"node-a"}, + }); code != http.StatusCreated { + t.Fatalf("restore into the prepared target = %d, want 201", code) + } + + assertDataPlaneStack(t, st, "pvc-prep") +} diff --git a/pkg/rest/rg_deleted_race.go b/pkg/rest/rg_deleted_race.go index 7a99fc3a..e9e0fa8e 100644 --- a/pkg/rest/rg_deleted_race.go +++ b/pkg/rest/rg_deleted_race.go @@ -21,6 +21,7 @@ package rest import ( "context" "fmt" + "net/http" "slices" "strings" "time" @@ -94,10 +95,12 @@ func (s *Server) parentRGSurvived(ctx context.Context, rgName string) (bool, err // groupRecheckBudget 0.6s parentRGSurvived's cache-retry // + 2 * cacheConvergeBudget 10s the rollback's two convergence waits // + rollbackWriteBudget 2s the rollback's own writes -// + 2 * markWriteBudget 2s the abandoned-rollback mark, before and after -// = detachedRollbackBudget 14.6s +// + 3 * markWriteBudget 3s the abandoned-rollback mark: in progress, the +// step it takes apart before the first delete, +// and the step it stopped at +// = detachedRollbackBudget 15.6s // + shutdownMargin 2s the rest of a graceful shutdown -// = gracefulShutdownWindow 16.6s how long Shutdown waits for in-flight handlers +// = gracefulShutdownWindow 17.6s how long Shutdown waits for in-flight handlers // + terminationGraceMargin 3s // <= terminationGracePeriodSeconds in every manifest that serves REST // @@ -113,7 +116,7 @@ const ( groupRecheckBudget = cacheRetryAttempts * cacheRetryDelay rollbackWriteBudget = 2 * time.Second markWriteBudget = time.Second - detachedRollbackBudget = groupRecheckBudget + 2*cacheConvergeBudget + rollbackWriteBudget + 2*markWriteBudget + detachedRollbackBudget = groupRecheckBudget + 2*cacheConvergeBudget + rollbackWriteBudget + 3*markWriteBudget shutdownMargin = 2 * time.Second terminationGraceMargin = 3 * time.Second ) @@ -152,33 +155,300 @@ func detachedCompensation(ctx context.Context) (context.Context, context.CancelF // half-torn leftover: the budget running out mid-cascade leaves no context to // write it on, and a killed process runs nothing at all. A failure then only // refines the mark to the step it stopped at, best-effort. -func (s *Server) rollBackCompensating(ctx context.Context, rdName string, placed []string) error { - s.markRollbackAbandoned(ctx, rdName, rollbackInProgress) +// +// The handshake with an adopting retry fails closed. Without the in-progress +// mark an adopter cannot see that the definition is being taken away, and +// without the read of the adoption mark the rollback cannot see that a retry +// answered for it; in either case deleting could take a definition somebody +// was just told is theirs, so nothing is deleted and the definition is left +// for the replay gate and the operator, named in the error. +// +// scope says whether a definition that already reads as finished is spared; +// see rollbackScope. +func (s *Server) rollBackCompensating( + ctx context.Context, rdName string, placed []string, scope rollbackScope, +) error { + err := s.markRollbackAbandoned(ctx, rdName, rollbackInProgress) + if errors.Is(err, store.ErrNotFound) { + // The definition this rollback was for is gone, so there is nothing of + // it left to remove. Deleting by name past this point would take + // whatever stands under the name by the time the cascade runs, and a + // retry recreating the deterministic name inside that window comes in + // as a creator, invisible to the handshake below. + return nil + } + + if err != nil { + // A write that failed at its deadline may still have landed; nothing + // is deleted either way, so the mark must not outlive this answer. + _ = s.clearRollbackMark(ctx, rdName) // the step's own error is the answer + + return newRollbackError(rollbackStepMark, err) + } + + // The creating half of the handshake claimAdoptedLeftover describes: a + // retry that adopted the definition, and may already have answered for + // it, owns it now. + adopted, err := s.adoptedElsewhere(ctx, rdName) + if err != nil { + // Nothing was taken away, so the definition is no more than an + // unfinished leftover, which a retry resumes. A mark naming this step + // would instead refuse every retry while the advice says to retry; + // the in-progress mark comes off for the same reason. + _ = s.clearRollbackMark(ctx, rdName) // the step's own error is the answer + + return newRollbackError(rollbackStepReadAdoption, err) + } + + if adopted { + return s.leftWhole(ctx, rdName, errRollbackYielded) + } + + if scope != rollbackEvenIfFinished { + finished, err := store.ReadsAsFinished(ctx, s.Store, rdName, scope == rollbackUnlessPlaced) + if err != nil { + _ = s.clearRollbackMark(ctx, rdName) // the step's own error is the answer + + return newRollbackError(rollbackStepReadFinished, err) + } + + if finished { + return s.leftWhole(ctx, rdName, errRollbackAnswered) + } + } - err := s.rollBackMaterialisedRD(ctx, rdName, placed) + // Nothing is touched until the snapshot refusal has run, for the reason + // handleRDDelete gives for its own: once the replicas are reaped, a + // refused definition delete leaves the target half torn down, with its + // children going and its parent kept, which no retry reconciles. A refusal + // whose correction is "drop the snapshots and retry" has to arrive while + // there is still something to retry over. + err = s.refuseRollbackOverSnapshots(ctx, rdName) if err != nil { - s.markRollbackAbandoned(ctx, rdName, rollbackStepName(err)) + _ = s.markRollbackAbandoned(ctx, rdName, rollbackStepName(err)) + + return err + } + + gone, err := s.enterDestructiveRollback(ctx, rdName) + if gone || err != nil { + return err + } + + err = s.rollBackMaterialisedRD(ctx, rdName, placed) + if err != nil { + _ = s.markRollbackAbandoned(ctx, rdName, rollbackStepName(err)) } return err } +// errRollbackMarkNotOurs is a definition that no longer carries the +// in-progress mark this rollback wrote: an operator cleared it, or the +// definition under the name is not the one the mark went on. +var errRollbackMarkNotOurs = errors.New("the rollback mark this rollback wrote is no longer on the definition") + +// enterDestructiveRollback replaces the in-progress mark with the first step +// that takes the definition apart, before that step runs. The in-progress +// mark is then only ever over a definition left whole, which is what lets an +// operator clear it (store.RollbackMarkClearRefusal): written after a failure +// instead, the step name could not land in the cases that tear the definition +// halfway, the budget running out mid-cascade or the process being killed, and +// in-progress would stand over half-reaped replicas. A process killed between +// this write and the first delete leaves a step name over a whole definition; +// that fails closed, to a delete by hand. +// +// The write checks the definition still carries this rollback's own mark, +// which is the identity check deleting by name needs: a definition recreated +// under the name since carries none. gone means there is nothing left to roll +// back. An error means nothing was deleted, and the in-progress mark is taken +// off unless it was not ours to begin with. +func (s *Server) enterDestructiveRollback(ctx context.Context, rdName string) (bool, error) { + markCtx, cancel := context.WithTimeout(ctx, markWriteBudget) + defer cancel() + + err := s.Store.ResourceDefinitions().PatchResourceDefinitionSpec(markCtx, rdName, + func(rd *apiv1.ResourceDefinition) error { + if rd.Props[rollbackAbandonedKey] != rollbackInProgress { + return errRollbackMarkNotOurs + } + + rd.Props[rollbackAbandonedKey] = rollbackStepNames[rollbackStepReapReplicas] + + return nil + }) + + switch { + case err == nil: + return false, nil + case errors.Is(err, store.ErrNotFound): + return true, nil + case errors.Is(err, errRollbackMarkNotOurs): + return false, newRollbackError(rollbackStepMark, fmt.Errorf("%q: %w", rdName, err)) + default: + _ = s.clearRollbackMark(ctx, rdName) // the step's own error is the answer + + return false, newRollbackError(rollbackStepMark, errors.Wrapf(err, "mark %q as being taken apart", rdName)) + } +} + +// finishedLeftoverRefusal takes the adoption mark on a finished leftover a +// replay is about to answer for. A replay writes nothing else, and a creator +// whose rollback judges "finished" by a replica a bare replay did not need +// would otherwise delete the definition after the 201. Nil means the replay may +// answer; a creator already rolling back is refused the way the gate refuses +// it, and a mark that cannot be written is a retryable 500. +func (s *Server) finishedLeftoverRefusal(ctx context.Context, door, rdName string) (int, *apiv1.APICallRc) { + err := s.claimAdoptedLeftover(ctx, rdName, nil) + if err == nil { + return 0, nil + } + + if errors.Is(err, errAdoptedLeftoverRollingBack) { + if status, refusal := s.abandonedRollbackRefusal(ctx, door, rdName); refusal != nil { + return status, refusal + } + + return http.StatusConflict, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "'" + rdName + "' is being rolled back by the attempt that created it", + Correc: "retry once the rollback has finished", + } + } + + return http.StatusInternalServerError, &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: "'" + rdName + "' is finished, but it could not be marked as answered for: " + + scrubImplDetails(err.Error()), + Correc: "retry the " + door, + } +} + +// errRollbackYielded reports a rollback that left the definition in place +// because a retry marked it adopted. That retry has answered for it, or its +// answer failed after its mark landed; either way nothing is deleted, and the +// next retry resumes the leftover. +var errRollbackYielded = errors.New("a retry marked the definition adopted, so it was left in place") + +// errRollbackAnswered reports a rollback that left the definition in place +// because it already reads as finished: the clone status poll answers such a +// definition without adopting it, so it may already have told the caller the +// operation completed. +var errRollbackAnswered = errors.New("the definition already reads as finished, so it was left in place") + +// rollbackScope is which definitions a rollback spares besides an adopted one. +// +// A failed materialisation spares one that already reads as finished +// (store.ReadsAsFinished): the clone status poll answers it as done the moment +// it is, without writing anything, which can be while the attempt that created it is still +// stamping its last replica, and deleting it afterwards takes a volume the +// driver was told exists. The RG-deleted rollback spares nothing: a finished +// definition parented to a group that is gone is what it exists to remove. +type rollbackScope int + +const ( + // rollbackEvenIfFinished deletes a finished definition too. + rollbackEvenIfFinished rollbackScope = iota + // rollbackUnlessPlaced spares one holding every volume and a replica: a + // clone, or a restore that was asked to place replicas. + rollbackUnlessPlaced + // rollbackUnlessHydrated spares one holding every volume: a bare restore, + // which places nothing and is finished without a replica. + rollbackUnlessHydrated +) + +// adoptedElsewhere reads, past the cache, whether a retry has adopted the +// definition. A definition that is gone was adopted by nobody. +func (s *Server) adoptedElsewhere(ctx context.Context, rdName string) (bool, error) { + return store.AdoptedElsewhere(ctx, s.Store, rdName) //nolint:wrapcheck // names the definition already +} + +// clearRollbackMark takes back the in-progress mark of a rollback that deleted +// nothing, so a retry is not refused over a definition that was left whole. +// +// It is tried more than once: a mark that stays is read by the replay gate as +// a rollback that never reported how it ended, which it has to refuse, since a +// process killed mid-cascade leaves exactly that. Each attempt has the mark's +// own budget. If every attempt fails the mark stays and the error is +// returned, so a definition left whole says so (leftWhole); the gate's +// refusal over the mark names clearing it as the way out. +func (s *Server) clearRollbackMark(ctx context.Context, rdName string) error { + const attempts = 3 + + var err error + + for range attempts { + err = s.patchRollbackMarkAway(ctx, rdName) + if err == nil || errors.Is(err, store.ErrNotFound) || ctx.Err() != nil { + break + } + } + + if err != nil && !errors.Is(err, store.ErrNotFound) { + log.FromContext(ctx).Info("could not clear the mark of a rollback that deleted nothing", + "resourceDefinition", rdName, "error", err.Error()) + + return err + } + + return nil +} + +// leftWhole takes the mark off a definition the rollback left whole for why, +// a yield to an adopter or a definition that already reads as finished. One +// whose mark stays is refused by every retry over a definition somebody was +// told is theirs, so the answer says so and names the command that clears it. +func (s *Server) leftWhole(ctx context.Context, rdName string, why error) error { + err := s.clearRollbackMark(ctx, rdName) + if err != nil { + return &rollbackMarkStuckError{why: why, err: err} + } + + return why +} + +// rollbackMarkStuckError is a rollback that left the definition whole, for +// why, and could not take its in-progress mark back off. +type rollbackMarkStuckError struct { + why, err error +} + +func (e *rollbackMarkStuckError) Error() string { + return e.why.Error() + ", but its rollback mark could not be taken off: " + e.err.Error() +} + +func (e *rollbackMarkStuckError) Unwrap() []error { return []error{e.why, e.err} } + +func (s *Server) patchRollbackMarkAway(ctx context.Context, rdName string) error { + markCtx, cancel := context.WithTimeout(ctx, markWriteBudget) + defer cancel() + + return s.Store.ResourceDefinitions().PatchResourceDefinitionSpec(markCtx, rdName, //nolint:wrapcheck // logged by the caller + func(rd *apiv1.ResourceDefinition) error { + delete(rd.Props, rollbackAbandonedKey) + + return nil + }) +} + // rollbackInProgress is the mark a rollback carries until it either completes, // taking the definition and the mark with it, or names the step it stopped at. // Read back, it is a rollback that never reported how it ended. -const rollbackInProgress = "in-progress" +const rollbackInProgress = store.RollbackInProgress // rollbackAbandonedKey marks a definition whose compensation gave up, with // the step it stopped at. // -// The replay gate needs it. A rollback that stops after placement succeeded -// on one node and failed on another leaves a definition holding fewer -// replicas than the operation intended, and the 500 advising a manual delete -// goes to a caller that retries under the same deterministic name long before -// anyone reads it: the retry finds volumes and a live replica and would answer -// 201 over it. Comparing the replicas with the snapshot's nodes cannot tell -// that apart from a finished clone that was evacuated or scaled down since, -// and the rollback is the one party that knows it gave up. +// The replay gate needs it. A rollback runs over a definition that may +// already hold volumes and a live replica: the RG-deleted rollback deletes a +// finished one by design, and a failed materialisation's rollback deletes one +// that does not read as finished yet. Cut short mid-cascade (its budget runs +// out, the process is killed, a replica or a snapshot cannot be reaped), it +// leaves a definition the 500 says to delete by hand, and that 500 goes to a +// caller that retries under the same deterministic name long before anyone +// reads it. The retry can find volumes and a live replica and would answer 201 +// over a definition the rollback was taking apart; the rollback is the one +// party that knows it gave up. const rollbackAbandonedKey = store.RollbackAbandonedProp // markRollbackAbandoned records on the definition that its rollback started, @@ -194,7 +464,7 @@ const rollbackAbandonedKey = store.RollbackAbandonedProp // resourceVersion a mark could otherwise spend the convergence waits the // cascade after it is budgeted for. A write abandoned at its deadline cannot // land later: every call it would still make runs on the expired context. -func (s *Server) markRollbackAbandoned(ctx context.Context, rdName, step string) { +func (s *Server) markRollbackAbandoned(ctx context.Context, rdName, step string) error { markCtx, cancel := context.WithTimeout(ctx, markWriteBudget) defer cancel() @@ -218,7 +488,7 @@ func (s *Server) markRollbackAbandoned(ctx context.Context, rdName, step string) select { case err = <-done: case <-markCtx.Done(): - err = markCtx.Err() + err = errors.Wrapf(markCtx.Err(), "mark %q", rdName) } switch { @@ -233,6 +503,8 @@ func (s *Server) markRollbackAbandoned(ctx context.Context, rdName, step string) log.FromContext(ctx).Info("could not mark an abandoned rollback on its definition", "resourceDefinition", rdName, "step", step, "reason", err.Error()) } + + return err } // failedMaterialiseRefusal rolls back what a failed materialisation left, when @@ -244,9 +516,18 @@ func (s *Server) markRollbackAbandoned(ctx context.Context, rdName, step string) // it. Only a failure materializeRestoredRD reports as after its own create is // rolled back; any other leaves whatever was there before the call, which may // belong to another attempt that is still running, and is never touched. +// +// wanted names the nodes the request asked for replicas on, when the caller +// places them itself: a definition left because it already reads as finished +// is not re-placed by a retry, so the replicas it never got are named with the +// command that places them. A clone passes none, since linstor-csi reconciles +// a clone's placement right after it reports complete. func (s *Server) failedMaterialiseRefusal( - ctx context.Context, message, noun, rdName string, placed []string, err error, + ctx context.Context, message, noun, rdName string, placed []string, scope rollbackScope, + wanted []string, err error, ) *apiv1.APICallRc { + message = scrubImplDetails(message) + var partial *materialiseAfterCreateError if !errors.As(err, &partial) { return &apiv1.APICallRc{RetCode: apiCallRcError, Message: message} @@ -255,7 +536,38 @@ func (s *Server) failedMaterialiseRefusal( rollbackCtx, cancel := detachedCompensation(ctx) defer cancel() - rollbackErr := s.rollBackCompensating(rollbackCtx, rdName, placed) + rollbackErr := s.rollBackCompensating(rollbackCtx, rdName, placed, scope) + if errors.Is(rollbackErr, errRollbackAnswered) { + cause, correc := rollbackFailureAdvice(rollbackErr, rdName) + + // A stuck mark's way out comes first; the replicas the request never + // got are still named after it, since the retry it leads to will + // not place them either. + var stuck *rollbackMarkStuckError + if short := s.answeredLeftShortCorrection(rollbackCtx, rdName, wanted, ""); short != "" { + if errors.As(rollbackErr, &stuck) { + correc += "; " + short + } else { + correc = short + } + } + + return &apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: message + "; '" + rdName + "' already reads as finished, so it was left in place", + Cause: cause, + Correc: correc, + } + } + + if errors.Is(rollbackErr, errRollbackYielded) { + cause, correc := rollbackFailureAdvice(rollbackErr, rdName) + left := "; '" + rdName + "' was left in place, because a retry of this " + noun + + " marked it adopted and may have answered for it" + + return &apiv1.APICallRc{RetCode: apiCallRcError, Message: message + left, Cause: cause, Correc: correc} + } + if rollbackErr != nil { cause, correc := rollbackFailureAdvice(rollbackErr, rdName) @@ -275,6 +587,30 @@ func (s *Server) failedMaterialiseRefusal( } } +// answeredLeftShortCorrection is the correction for a definition the rollback +// left because it already reads as finished, when the request asked for +// replicas on wanted: a retry judges it finished too and places nothing, so a +// replica that never landed is named with the command that places it. correc +// is the answer when every one of them landed. +func (s *Server) answeredLeftShortCorrection(ctx context.Context, rdName string, wanted []string, correc string) string { + if len(wanted) == 0 { + return correc + } + + missing, err := store.MissingReplicas(ctx, s.Store, rdName, wanted) + + switch { + case err != nil: + return "check which of the requested replicas of '" + rdName + "' exist, and place the rest " + + "with `linstor resource create " + rdName + "`" + case len(missing) > 0: + return "'" + rdName + "' has no replica on " + strings.Join(missing, ", ") + + ", which a retry will not place: run `linstor resource create " + rdName + "` for each" + default: + return correc + } +} + // adoptedOverDeletedGroupRefusal is the RG-deleted refusal over a definition // this request did not create. It is left in place: the leftover belongs to an // earlier attempt at the same operation, which may still be running, and @@ -315,6 +651,9 @@ const ( rollbackStepSnapshots rollbackStepDeleteDefinition rollbackStepReadSnapshots + rollbackStepMark + rollbackStepReadAdoption + rollbackStepReadFinished ) type rollbackStepError struct { @@ -332,11 +671,14 @@ func newRollbackError(step rollbackStep, err error) error { // rollbackStepNames spells each step for the abandoned-rollback mark. var rollbackStepNames = map[rollbackStep]string{ //nolint:gochecknoglobals // a fixed table, read-only - rollbackStepReapReplicas: "reap-replicas", + rollbackStepReapReplicas: store.RollbackStepReapReplicas, rollbackStepRereadReplicas: "reread-replicas", - rollbackStepSnapshots: "snapshots", - rollbackStepDeleteDefinition: "delete-definition", - rollbackStepReadSnapshots: "read-snapshots", + rollbackStepSnapshots: store.RollbackStepSnapshots, + rollbackStepDeleteDefinition: store.RollbackStepDeleteDefinition, + rollbackStepReadSnapshots: store.RollbackStepReadSnapshots, + rollbackStepMark: "mark", + rollbackStepReadAdoption: "read-adoption", + rollbackStepReadFinished: "read-finished", } // rollbackStepName spells the step a compensation stopped at, or "unknown". @@ -360,9 +702,81 @@ func rollbackStepByName(name string) (rollbackStep, bool) { return rollbackStepReapReplicas, false } +// retryOrDeleteByHand is the correction for a rollback that deleted nothing. +// The retry resumes the definition; the way out is named as well, since a mark +// the rollback could not take back makes the replay gate refuse that retry. +func retryOrDeleteByHand(rdName string) string { + return "retry: the retry resumes '" + rdName + "' or reports it finished; if the retry is " + + "refused over an abandoned rollback, delete '" + rdName + "' by hand" +} + +// rollbackLeftNothingDeleted reports a rollback that, by design, deleted +// nothing: it yielded to an adopting retry or could not hold the handshake. +func rollbackLeftNothingDeleted(err error) bool { + if errors.Is(err, errRollbackYielded) || errors.Is(err, errRollbackAnswered) { + return true + } + + var failure *rollbackStepError + + return errors.As(err, &failure) && + (failure.step == rollbackStepMark || failure.step == rollbackStepReadAdoption || + failure.step == rollbackStepReadFinished) +} + +// rollbackFailureAdviceOverDeletedGroup is rollbackFailureAdvice for the +// doors whose rollback runs because the definition's group was deleted. A +// rollback that deleted nothing there leaves a definition parented to a group +// that is gone, and a plain retry meets the group refusal rather than a +// resume, so the correction starts with the group. +func rollbackFailureAdviceOverDeletedGroup(err error, rdName, rgName string) (string, string) { + cause, correc := rollbackFailureAdvice(err, rdName) + if !rollbackLeftNothingDeleted(err) { + return cause, correc + } + + // The mark it could not take off refuses the retry the group's return + // would otherwise let through, so clearing it comes between the two. + var stuck *rollbackMarkStuckError + if errors.As(err, &stuck) { + return cause, "re-create resource group '" + rgName + "', " + clearRollbackMarkCommand(rdName) + + ", then retry, which resumes '" + rdName + "'" + } + + return cause, "re-create resource group '" + rgName + "' and retry, which resumes '" + rdName + + "'; or delete '" + rdName + "' by hand" +} + // rollbackFailureAdvice is the Cause and Correc for a failed compensation, // written for the step that failed rather than once for all of them. func rollbackFailureAdvice(err error, rdName string) (string, string) { + var stuck *rollbackMarkStuckError + if errors.As(err, &stuck) { + return "'" + rdName + "' was left in place, but the mark saying it is being rolled back " + + "could not be taken off, so every retry is refused over it", + clearRollbackMarkCommand(rdName) + ", then retry" + } + + if errors.Is(err, errRollbackAnswered) { + return "'" + rdName + "' already held everything a retry or a status poll reads as " + + "finished when this attempt failed, so one of them may already have reported it " + + "complete, and it was left in place", + "retry, which answers for '" + rdName + "' as it stands" + } + + if errors.Is(err, errRollbackMarkNotOurs) { + return "the rollback's mark on '" + rdName + "' was cleared or replaced before it took " + + "anything apart, so it may no longer be the definition the rollback set out to " + + "remove, and it was left in place", + retryOrDeleteByHand(rdName) + } + + if errors.Is(err, errRollbackYielded) { + return "a retry of the same operation marked '" + rdName + "' adopted while this attempt " + + "failed, and may already have answered for it, so it was left in place", + retryOrDeleteByHand(rdName) + } + var failure *rollbackStepError if !errors.As(err, &failure) { return rollbackStepAdvice(rollbackStepReapReplicas, false, rdName) @@ -388,15 +802,32 @@ func rollbackStepAdvice(step rollbackStep, known bool, rdName string) (string, s return "a snapshot exists on the definition, and the rollback does not destroy " + "a snapshot the way `rd d` refuses to", "delete the snapshot(s) of '" + rdName + "' if they are not needed, then delete '" + - rdName + "' by hand" + rdName + "' by hand; or, since the rollback touched nothing, " + + clearRollbackMarkCommand(rdName) + " to keep it as it stands" case rollbackStepReadSnapshots: return "the snapshots of the definition could not be read, and the rollback does " + "not delete a definition it cannot show has none", "check whether '" + rdName + "' has snapshots (`linstor s l`), delete any that are " + - "not needed, then delete '" + rdName + "' by hand" + "not needed, then delete '" + rdName + "' by hand; or, since the rollback touched " + + "nothing, " + clearRollbackMarkCommand(rdName) + " to keep it as it stands" case rollbackStepDeleteDefinition: return "every replica went, but deleting the definition itself failed", "delete '" + rdName + "' by hand" + case rollbackStepMark: + return "the rollback could not record that it started, so a retry of the same " + + "operation could not have seen it, and the definition was left in place " + + "rather than taken from under a retry that may have answered for it", + retryOrDeleteByHand(rdName) + case rollbackStepReadAdoption: + return "the rollback could not read whether a retry had adopted the definition, " + + "so it was left in place rather than taken from under one that may have " + + "answered for it", + retryOrDeleteByHand(rdName) + case rollbackStepReadFinished: + return "the rollback could not read the volumes and replicas of the definition to tell " + + "whether a retry or a status poll had already been told it is complete, so it was " + + "left in place rather than taken from under one that may have answered for it", + retryOrDeleteByHand(rdName) case rollbackStepReapReplicas: } @@ -435,17 +866,11 @@ func rollbackStepAdvice(step rollbackStep, known bool, rdName string) (string, s // So this returns an error, and a caller that gets one must not report a // rollback. What is left behind is a definition parented to a group that is // gone, which is the state the operator has to be told about, with its name. +// +// The caller has already run the snapshot refusal and marked the definition +// as being taken apart (enterDestructiveRollback); everything here deletes. func (s *Server) rollBackMaterialisedRD(ctx context.Context, rdName string, placed []string) error { - // Nothing is touched until the snapshot refusal has run, for the reason - // handleRDDelete gives for its own: once the replicas are reaped, a - // refused definition delete leaves the target half torn down, with its - // children going and its parent kept, which no retry reconciles. A refusal - // whose correction is "drop the snapshots and retry" has to arrive while - // there is still something to retry over. - err := s.refuseRollbackOverSnapshots(ctx, rdName) - if err != nil { - return err - } + var err error // The replicas this request placed are deleted by name first. A write goes // to the API server whatever the cache has seen, so this is the one step diff --git a/pkg/rest/rg_deleted_race_round10_test.go b/pkg/rest/rg_deleted_race_round10_test.go index 345fa64c..55c0f402 100644 --- a/pkg/rest/rg_deleted_race_round10_test.go +++ b/pkg/rest/rg_deleted_race_round10_test.go @@ -5,8 +5,6 @@ package rest import ( "context" "net/http" - "net/http/httptest" - "strings" "sync" "testing" "time" @@ -15,10 +13,9 @@ import ( "github.com/cozystack/blockstor/pkg/store" ) -// Every door stamps a group, so the grouped branch of the abandoned-rollback -// gate is the mainline one, and the round-9 fixture reached only the -// ungrouped branch. -func TestRDCloneReplayRefusesAGroupedLeftoverWhoseRollbackGaveUp(t *testing.T) { +// Every door stamps a group, so the grouped leftover is the mainline one, and +// the round-9 fixture reached only the ungrouped branch. +func TestRDCloneGroupedHalfPlacedByAFailedAttemptIsLeftForTheReplay(t *testing.T) { t.Parallel() backend := store.NewInMemory() @@ -45,24 +42,15 @@ func TestRDCloneReplayRefusesAGroupedLeftoverWhoseRollbackGaveUp(t *testing.T) { _ = first.Body.Close() if first.StatusCode != http.StatusInternalServerError { - t.Fatalf("first attempt = %d, want 500: placement failed and the rollback gave up", first.StatusCode) + t.Fatalf("first attempt = %d, want 500: placement failed on node-b", first.StatusCode) } leftover, err := backend.ResourceDefinitions().Get(t.Context(), "dst-half10") if err != nil || leftover.ResourceGroupName != "grp-half10" { - t.Fatalf("fixture: want a grouped leftover, got group %q (err=%v)", leftover.ResourceGroupName, err) + t.Fatalf("want the grouped leftover left in place, got group %q (err=%v)", leftover.ResourceGroupName, err) } - retry := postClone(t, base, "src-half10", map[string]any{"name": "dst-half10", "use_zfs_clone": true}) - defer func() { _ = retry.Body.Close() }() - - if retry.StatusCode == http.StatusCreated { - t.Fatal("the retry answered 201 over a grouped leftover whose rollback gave up half-placed") - } - - if rc := decodeCloneMessage(t, retry); !strings.Contains(rc.Message, "rollback gave up") { - t.Errorf("refusal %q does not say an earlier rollback gave up", rc.Message) - } + assertHalfPlacedCloneLeftForTheReplay(t, backend, base, "src-half10", "dst-half10") } // deadlineResources blocks a replica delete until its context ends, the way a @@ -115,7 +103,7 @@ func TestAnAbandonedRollbackIsMarkedEvenWhenItsBudgetRunsOut(t *testing.T) { rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 300*time.Millisecond) defer cancel() - if err := s.rollBackCompensating(rollbackCtx, "dst-budget10", []string{"node-a"}); err == nil { + if err := s.rollBackCompensating(rollbackCtx, "dst-budget10", []string{"node-a"}, rollbackEvenIfFinished); err == nil { t.Fatal("fixture: the rollback was supposed to run out of budget") } @@ -128,8 +116,7 @@ func TestAnAbandonedRollbackIsMarkedEvenWhenItsBudgetRunsOut(t *testing.T) { t.Fatal("a rollback that ran out of budget mid-cascade left no mark, so the replay would answer 201 over it") } - w := httptest.NewRecorder() - if !(&Server{Store: backend}).cloneRollbackWasAbandoned(ctx, w, "src", "dst-budget10") { + if _, refusal := (&Server{Store: backend}).abandonedRollbackRefusal(ctx, "clone", "dst-budget10"); refusal == nil { t.Error("the replay gate did not refuse over the mark") } } @@ -188,7 +175,7 @@ func TestTheRollbackRunsOnWhatIsLeftOfTheDoorsBudget(t *testing.T) { doorDeadline, _ := doorCtx.Deadline() - _ = s.rollBackCompensating(doorCtx, "dst-chain10", nil) + _ = s.rollBackCompensating(doorCtx, "dst-chain10", nil, rollbackEvenIfFinished) if seen.IsZero() { t.Fatal("fixture: the rollback's context carried no deadline at all") diff --git a/pkg/rest/rg_deleted_race_round5_test.go b/pkg/rest/rg_deleted_race_round5_test.go index 1ad1ab09..9afd7311 100644 --- a/pkg/rest/rg_deleted_race_round5_test.go +++ b/pkg/rest/rg_deleted_race_round5_test.go @@ -243,7 +243,7 @@ func TestRDCloneRollbackRefusesOverASnapshotOnTheTarget(t *testing.T) { // A failed rollback may reap every replica and still keep the definition, and // both corrections tell the operator to re-create the group. The moment they // do, a gate asking only about the group answered 201 for a clone on no node. -func TestRDCloneReplayRefusesALeftoverWhoseReplicasWereReaped(t *testing.T) { +func TestRDCloneRetryOverAReapedLeftoverAnswersOnlyAWholeClone(t *testing.T) { t.Parallel() st := store.NewInMemory() @@ -271,9 +271,14 @@ func TestRDCloneReplayRefusesALeftoverWhoseReplicasWereReaped(t *testing.T) { resp := postClone(t, base, "src-reaped", map[string]any{"name": "dst-reaped", "use_zfs_clone": true}) _ = resp.Body.Close() - if resp.StatusCode == http.StatusCreated { - t.Error("replay answered 201 for a clone whose replicas were all reaped") + // With no abandoned-rollback mark on it, the leftover is an unfinished + // clone and the retry resumes it. A 201 is fine; one over a clone that + // still exists on no node is not. + if resp.StatusCode != http.StatusCreated { + t.Fatalf("retry over an unmarked, reaped leftover = %d, want 201 over a resumed clone", resp.StatusCode) } + + assertCloneWhole(t, st, "dst-reaped") } // Refusing on an unreadable group costs nothing, since the CSI retry heals diff --git a/pkg/rest/rg_deleted_race_round6_test.go b/pkg/rest/rg_deleted_race_round6_test.go index 62782777..01a32a7d 100644 --- a/pkg/rest/rg_deleted_race_round6_test.go +++ b/pkg/rest/rg_deleted_race_round6_test.go @@ -22,7 +22,7 @@ import ( // seedCloneLeftover writes a definition carrying the clone marker of // src→dst straight into the store, with a volume when withVolume is set and // one replica on node-a when replicaFlags is non-nil. -func seedCloneLeftover(t *testing.T, st store.Store, src, dst string, withVolume bool, replicaFlags []string) { +func seedShapedCloneLeftover(t *testing.T, st store.Store, src, dst string, withVolume bool, replicaFlags []string) { t.Helper() ctx := t.Context() @@ -125,7 +125,7 @@ func TestRDCloneReplayOverASeededLeftoverTurnsOnTheDeletionStamp(t *testing.T) { st := store.NewInMemory() seedDeployedCloneSource(t, st, "src-seed") - seedCloneLeftover(t, st, "src-seed", tc.dstName, true, tc.flags) + seedShapedCloneLeftover(t, st, "src-seed", tc.dstName, true, tc.flags) base, stop := startServerWithStore(t, st) defer stop() @@ -140,57 +140,78 @@ func TestRDCloneReplayOverASeededLeftoverTurnsOnTheDeletionStamp(t *testing.T) { } } +// assertCloneWhole fails unless the clone holds a volume and a replica that is +// not being deleted: the least a 201 may be answered over. +func assertCloneWhole(t *testing.T, st store.Store, cloneName string) { + t.Helper() + + vds, err := st.VolumeDefinitions().List(t.Context(), cloneName) + if err != nil || len(vds) == 0 { + t.Errorf("201 over a clone with no volumes (%v)", err) + } + + replicas, err := st.Resources().ListByDefinition(t.Context(), cloneName) + if err != nil { + t.Fatalf("list the clone's replicas: %v", err) + } + + for i := range replicas { + if !replicaAcceptedForDeletion(&replicas[i]) { + return + } + } + + t.Errorf("201 over a clone with no live replica") +} + // Only the replicas term of the wholeness check had a fixture. A leftover with -// a replica and no volume isolates the volumes term. -func TestRDCloneReplayRefusesALeftoverWithReplicasButNoVolumes(t *testing.T) { +// a replica and no volume isolates the volumes term. A retry resumes it rather +// than refusing it, and the 201 it then answers is over a clone the resume made +// whole. +func TestRDCloneRetryMakesWholeALeftoverWithReplicasButNoVolumes(t *testing.T) { t.Parallel() st := store.NewInMemory() seedDeployedCloneSource(t, st, "src-novol") - seedCloneLeftover(t, st, "src-novol", "dst-novol", false, []string{}) + seedShapedCloneLeftover(t, st, "src-novol", "dst-novol", false, []string{}) base, stop := startServerWithStore(t, st) defer stop() resp := postClone(t, base, "src-novol", map[string]any{"name": "dst-novol", "use_zfs_clone": true}) - defer func() { _ = resp.Body.Close() }() + _ = resp.Body.Close() - if resp.StatusCode == http.StatusCreated { - t.Fatal("replay = 201 over a leftover with no volumes") + // A live replica is what tells this leftover from one being torn down, + // and the resume has to finish it: a refusal here strands the CSI target + // name for good, since nobody is deleting the replica it waits for. + if resp.StatusCode != http.StatusCreated { + t.Fatalf("retry over a volume-less leftover with a live replica = %d, want 201", resp.StatusCode) } - if rc := decodeCloneMessage(t, resp); !strings.Contains(rc.Cause, "no volumes") { - t.Errorf("cause = %q, want it to name the missing volumes", rc.Cause) - } + assertCloneWhole(t, st, "dst-novol") } -// A leftover that stopped before creating any volume is not rollback debris, -// and the refusal must not tell a first attempt still running to delete itself. -func TestRDCloneReplayWordsAMarkerOnlyLeftoverForBothWaysItArises(t *testing.T) { +// A leftover that stopped before creating any volume is not rollback debris: +// it is an attempt that got as far as the marker, and a retry finishes it. The +// 201 is answered only over what the resume made whole. +func TestRDCloneRetryFinishesAMarkerOnlyLeftover(t *testing.T) { t.Parallel() st := store.NewInMemory() seedDeployedCloneSource(t, st, "src-bare") - seedCloneLeftover(t, st, "src-bare", "dst-bare", false, nil) + seedShapedCloneLeftover(t, st, "src-bare", "dst-bare", false, nil) base, stop := startServerWithStore(t, st) defer stop() resp := postClone(t, base, "src-bare", map[string]any{"name": "dst-bare", "use_zfs_clone": true}) - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode == http.StatusCreated { - t.Fatal("replay = 201 over a definition with no volumes and no replicas") - } + _ = resp.Body.Close() - rc := decodeCloneMessage(t, resp) - if strings.Contains(rc.Cause, "rollback") { - t.Errorf("cause = %q, which blames a rollback that never ran", rc.Cause) + if resp.StatusCode != http.StatusCreated { + t.Fatalf("retry over a marker-only leftover = %d, want the resume's 201", resp.StatusCode) } - if !strings.Contains(rc.Correc, "still running") { - t.Errorf("correc = %q, want it conditioned on no attempt still running", rc.Correc) - } + assertCloneWhole(t, st, "dst-bare") } // trailingReplicaListing hides the target's replicas from the first few @@ -544,7 +565,7 @@ func TestRDCloneFailureDoesNotRollBackADefinitionItDidNotCreate(t *testing.T) { backend := store.NewInMemory() ctx := t.Context() seedDeployedCloneSource(t, backend, "src-other") - seedCloneLeftover(t, backend, "src-other", "dst-other", false, nil) + seedShapedCloneLeftover(t, backend, "src-other", "dst-other", false, nil) blips := &atomic.Int32{} blips.Store(1) @@ -555,10 +576,9 @@ func TestRDCloneFailureDoesNotRollBackADefinitionItDidNotCreate(t *testing.T) { resp := postClone(t, base, "src-other", map[string]any{"name": "dst-other", "use_zfs_clone": true}) _ = resp.Body.Close() - if resp.StatusCode == http.StatusCreated { - t.Fatalf("status = 201, but the create under an existing name cannot have succeeded") - } - + // The leftover carries this clone's marker, so the create's AlreadyExists + // is the resume's to adopt, and a 201 may follow. What must not follow is + // a rollback of it. if _, err := backend.ResourceDefinitions().Get(ctx, "dst-other"); err != nil { t.Errorf("a definition this request did not create was deleted: %v", err) } diff --git a/pkg/rest/rg_deleted_race_round8_test.go b/pkg/rest/rg_deleted_race_round8_test.go index b456025c..4a979f94 100644 --- a/pkg/rest/rg_deleted_race_round8_test.go +++ b/pkg/rest/rg_deleted_race_round8_test.go @@ -111,15 +111,10 @@ func TestRDCloneShellRollbackOutlivesTheRequest(t *testing.T) { func TestRollbackBudgetFitsTheShutdownWindow(t *testing.T) { t.Parallel() - if detachedRollbackBudget+shutdownMargin > gracefulShutdownWindow { - t.Errorf("rollback budget %s plus margin %s does not fit the shutdown window %s", - detachedRollbackBudget, shutdownMargin, gracefulShutdownWindow) - } - - // The two mark writes are budgeted on top of everything the cascade is, + // The three mark writes are budgeted on top of everything the cascade is, // not carved out of its waits. - if cascade := groupRecheckBudget + 2*cacheConvergeBudget + rollbackWriteBudget; detachedRollbackBudget < cascade+2*markWriteBudget { - t.Errorf("rollback budget %s leaves the two abandoned-rollback marks (%s each) nothing on top "+ + if cascade := groupRecheckBudget + 2*cacheConvergeBudget + rollbackWriteBudget; detachedRollbackBudget < cascade+3*markWriteBudget { + t.Errorf("rollback budget %s leaves the three abandoned-rollback marks (%s each) nothing on top "+ "of the cascade's %s", detachedRollbackBudget, markWriteBudget, cascade) } @@ -143,6 +138,19 @@ func TestRollbackBudgetFitsTheShutdownWindow(t *testing.T) { t.Errorf("%s gives the pod %s; the shutdown window %s plus margin %s needs more", manifest, grace, gracefulShutdownWindow, terminationGraceMargin) } + + // The detached budget against the pod's grace on its own, so it is + // not held only by the window that is derived from it. + if detachedRollbackBudget+shutdownMargin+terminationGraceMargin > grace { + t.Errorf("%s gives the pod %s; the rollback budget %s plus margins %s and %s needs more", + manifest, grace, detachedRollbackBudget, shutdownMargin, terminationGraceMargin) + } + + // A claim's detached read-back also runs past its request. + if store.ClaimReadBackBudget+shutdownMargin+terminationGraceMargin > grace { + t.Errorf("%s gives the pod %s; the claim read-back %s plus margins needs more", + manifest, grace, store.ClaimReadBackBudget) + } } } diff --git a/pkg/rest/rg_deleted_race_round9_test.go b/pkg/rest/rg_deleted_race_round9_test.go index ec116c83..c29e37bf 100644 --- a/pkg/rest/rg_deleted_race_round9_test.go +++ b/pkg/rest/rg_deleted_race_round9_test.go @@ -404,13 +404,14 @@ func (s halfPlacingStore) Resources() store.ResourceStore { return halfPlacingResources{ResourceStore: s.Store.Resources(), target: s.target} } -// Placement succeeds on one node and fails on the other, the rollback gives up -// on a conflict, and linstor-csi retries under the same name long before -// anyone reads the 500. The leftover has its volumes and a live replica, so -// the wholeness gate answered 201 "already cloned" over a clone with fewer -// replicas than intended. The rollback records that it gave up, and the replay -// refuses on that record. -func TestRDCloneReplayRefusesALeftoverWhoseRollbackGaveUp(t *testing.T) { +// Placement succeeds on one node and fails on the other, and linstor-csi +// retries under the same name long before anyone reads the 500. The leftover +// has its volumes and a live replica, which is what the replay and the status +// poll answer as finished, and either may already have, while the second stamp +// was still in flight. So the first attempt leaves it rather than rolling back +// a clone somebody was told exists, leaves no mark that would refuse the +// retry, and topping up the placement is the reconciliation's job. +func TestRDCloneHalfPlacedByAFailedAttemptIsLeftForTheReplay(t *testing.T) { t.Parallel() backend := store.NewInMemory() @@ -423,22 +424,36 @@ func TestRDCloneReplayRefusesALeftoverWhoseRollbackGaveUp(t *testing.T) { _ = first.Body.Close() if first.StatusCode != http.StatusInternalServerError { - t.Fatalf("first attempt = %d, want 500: placement failed and the rollback gave up", first.StatusCode) + t.Fatalf("first attempt = %d, want 500: placement failed on node-b", first.StatusCode) } - replicas, err := backend.Resources().ListByDefinition(t.Context(), "dst-half9") - if err != nil || len(replicas) != 1 { - t.Fatalf("fixture: want the half-placed leftover with one replica, got %d (err=%v)", len(replicas), err) + assertHalfPlacedCloneLeftForTheReplay(t, backend, base, "src-half9", "dst-half9") +} + +// assertHalfPlacedCloneLeftForTheReplay checks a clone whose second stamp +// failed is still there with its one replica and no rollback mark, and that +// the replay answers it finished. +func assertHalfPlacedCloneLeftForTheReplay(t *testing.T, backend store.Store, base, src, dst string) { + t.Helper() + + rd, err := backend.ResourceDefinitions().Get(t.Context(), dst) + if err != nil { + t.Fatalf("the half-placed clone was rolled back: %v", err) } - retry := postClone(t, base, "src-half9", map[string]any{"name": "dst-half9", "use_zfs_clone": true}) - defer func() { _ = retry.Body.Close() }() + if mark := rd.Props[store.RollbackAbandonedProp]; mark != "" { + t.Errorf("the half-placed clone carries rollback mark %q, which refuses every retry", mark) + } - if retry.StatusCode == http.StatusCreated { - t.Fatal("the retry answered 201 over a leftover whose rollback gave up half-placed") + replicas, err := backend.Resources().ListByDefinition(t.Context(), dst) + if err != nil || len(replicas) != 1 { + t.Fatalf("want the one replica the first attempt placed, got %d (err=%v)", len(replicas), err) } - if rc := decodeCloneMessage(t, retry); !strings.Contains(rc.Message, "rollback gave up") { - t.Errorf("refusal %q does not say an earlier rollback gave up", rc.Message) + retry := postClone(t, base, src, map[string]any{"name": dst, "use_zfs_clone": true}) + defer func() { _ = retry.Body.Close() }() + + if retry.StatusCode != http.StatusCreated { + t.Errorf("replay of the half-placed clone = %d, want 201", retry.StatusCode) } } diff --git a/pkg/rest/rollback_handshake_failclosed_test.go b/pkg/rest/rollback_handshake_failclosed_test.go new file mode 100644 index 00000000..09b066f8 --- /dev/null +++ b/pkg/rest/rollback_handshake_failclosed_test.go @@ -0,0 +1,664 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +var ( + errPatchRefused = errors.New("patch refused") + errLiveReadRefused = errors.New("live read refused") +) + +// handshakeBlind fails the writes or the live reads the adopt/rollback +// handshake depends on. +type handshakeBlind struct { + store.Store + + failPatch, failLiveRead bool +} + +type handshakeBlindRDs struct { + store.ResourceDefinitionStore + + outer handshakeBlind +} + +func (h handshakeBlind) ResourceDefinitions() store.ResourceDefinitionStore { + return handshakeBlindRDs{ResourceDefinitionStore: h.Store.ResourceDefinitions(), outer: h} +} + +func (r handshakeBlindRDs) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + if r.outer.failPatch { + return errPatchRefused + } + + return r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) //nolint:wrapcheck // pass-through test double +} + +func (r handshakeBlindRDs) GetUncached(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + if r.outer.failLiveRead { + return apiv1.ResourceDefinition{}, errLiveReadRefused + } + + return r.ResourceDefinitionStore.GetUncached(ctx, name) //nolint:wrapcheck // pass-through test double +} + +func seedRolledBackDefinition(t *testing.T, st store.Store, name string) { + t.Helper() + + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{Name: name}); err != nil { + t.Fatalf("seed the definition: %v", err) + } + + if err := st.Resources().Create(t.Context(), &apiv1.Resource{Name: name, NodeName: "node-a"}); err != nil { + t.Fatalf("seed its replica: %v", err) + } +} + +// Without its in-progress mark an adopter cannot see the definition is being +// taken away, and without the read of the adoption mark the rollback cannot +// see a retry that answered for it. Either way it must not delete. +func TestTheRollbackDeletesNothingWhenItCannotHoldTheHandshake(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + blind handshakeBlind + wantStep string + }{ + {name: "mark-not-written", blind: handshakeBlind{failPatch: true}, wantStep: "mark"}, + {name: "adoption-not-read", blind: handshakeBlind{failLiveRead: true}, wantStep: "read-adoption"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedRolledBackDefinition(t, backend, "dst-blind") + + blind := tc.blind + blind.Store = backend + srv := &Server{Store: blind} + + err := srv.rollBackCompensating(t.Context(), "dst-blind", []string{"node-a"}, rollbackEvenIfFinished) + if err == nil { + t.Fatal("the rollback reported success without the handshake") + } + + if got := rollbackStepName(err); got != tc.wantStep { + t.Errorf("step = %q, want %q", got, tc.wantStep) + } + + if _, err := backend.ResourceDefinitions().Get(t.Context(), "dst-blind"); err != nil { + t.Errorf("the definition was deleted: %v", err) + } + + if replicas, _ := backend.Resources().ListByDefinition(t.Context(), "dst-blind"); len(replicas) != 1 || + replicaAcceptedForDeletion(&replicas[0]) { + t.Errorf("the replica was reaped: %v", replicas) + } + }) + } +} + +// landsThenFails applies the first patch and reports it failed, the way a +// write that landed just as its deadline ran out reads to its caller. +type landsThenFails struct { + store.Store + + failed *bool +} + +type landsThenFailsRDs struct { + store.ResourceDefinitionStore + + outer landsThenFails +} + +func (l landsThenFails) ResourceDefinitions() store.ResourceDefinitionStore { + return landsThenFailsRDs{ResourceDefinitionStore: l.Store.ResourceDefinitions(), outer: l} +} + +func (r landsThenFailsRDs) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + err := r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) + if err == nil && !*r.outer.failed { + *r.outer.failed = true + + return errPatchRefused + } + + return err //nolint:wrapcheck // pass-through test double +} + +// patchFailsOnce fails one numbered patch and lets the rest through, on top of +// a store whose live reads fail: the rollback's mark lands, the adoption read +// fails, and the first attempt to take the mark back fails with it. +type patchFailsOnce struct { + handshakeBlind + + calls *int + failAt int +} + +type patchFailsOnceRDs struct { + store.ResourceDefinitionStore + + outer patchFailsOnce +} + +func (p patchFailsOnce) ResourceDefinitions() store.ResourceDefinitionStore { + return patchFailsOnceRDs{ResourceDefinitionStore: p.handshakeBlind.ResourceDefinitions(), outer: p} +} + +func (r patchFailsOnceRDs) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + *r.outer.calls++ + if *r.outer.calls == r.outer.failAt { + return errPatchRefused + } + + return r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) //nolint:wrapcheck // pass-through test double +} + +// A rollback that deleted nothing leaves an unfinished leftover, which a retry +// resumes. A mark left on it made the replay gate refuse every retry while the +// correction said to retry, so linstor-csi looped on the same 409 for good. +func TestARollbackThatDeletedNothingLeavesNoMarkToRefuseTheRetry(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + build func(store.Store) store.Store + }{ + {name: "adoption-not-read", build: func(b store.Store) store.Store { + return handshakeBlind{Store: b, failLiveRead: true} + }}, + {name: "mark-landed-but-reported-failed", build: func(b store.Store) store.Store { + failed := false + + return landsThenFails{Store: b, failed: &failed} + }}, + {name: "first-clear-failed", build: func(b store.Store) store.Store { + calls := 0 + + return patchFailsOnce{handshakeBlind: handshakeBlind{Store: b, failLiveRead: true}, calls: &calls, failAt: 2} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + seedRolledBackDefinition(t, backend, "dst-nomark") + + srv := &Server{Store: tc.build(backend)} + + if err := srv.rollBackCompensating(ctx, "dst-nomark", []string{"node-a"}, rollbackEvenIfFinished); err == nil { + t.Fatal("the rollback reported success without the handshake") + } + + left, err := backend.ResourceDefinitions().Get(ctx, "dst-nomark") + if err != nil { + t.Fatalf("the definition was deleted: %v", err) + } + + if mark, ok := left.Props[rollbackAbandonedKey]; ok { + t.Errorf("the definition carries the abandoned-rollback mark %q", mark) + } + + gate := &Server{Store: backend} + for retry := range 2 { + if _, refusal := gate.abandonedRollbackRefusal(ctx, "clone", "dst-nomark"); refusal != nil { + t.Errorf("retry %d refused: %s; %s", retry, refusal.Message, refusal.Correc) + } + } + }) + } +} + +// A mark the rollback could not take back stays, and the gate refuses over it. +// Its correction has to lead out rather than repeat the retry that just failed. +func TestAMarkThatCouldNotBeClearedNamesTheWayOut(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-stuck", Props: map[string]string{rollbackAbandonedKey: rollbackInProgress}, + }); err != nil { + t.Fatalf("seed the definition: %v", err) + } + + _, refusal := (&Server{Store: backend}).abandonedRollbackRefusal(ctx, "clone", "dst-stuck") + if refusal == nil { + t.Fatal("the gate let a retry through an in-progress mark") + } + + if !strings.Contains(refusal.Correc, "by hand") { + t.Errorf("correction %q does not name the way out", refusal.Correc) + } +} + +// The corrections for a rollback that deleted nothing name the way out as +// well, and over a deleted group they start with the group, since a plain +// retry meets the group refusal rather than a resume. +func TestTheCorrectionForARollbackThatDeletedNothing(t *testing.T) { + t.Parallel() + + for _, err := range []error{ + errRollbackYielded, + newRollbackError(rollbackStepMark, errPatchRefused), + newRollbackError(rollbackStepReadAdoption, errLiveReadRefused), + } { + _, plain := rollbackFailureAdvice(err, "dst-x") + if !strings.Contains(plain, "delete 'dst-x' by hand") { + t.Errorf("%v: correction %q does not name the way out", err, plain) + } + + _, overGroup := rollbackFailureAdviceOverDeletedGroup(err, "dst-x", "grp-x") + if !strings.HasPrefix(overGroup, "re-create resource group 'grp-x'") { + t.Errorf("%v: correction over a deleted group %q does not start with the group", err, overGroup) + } + } +} + +// A rollback that cannot read whether the definition already reads as +// finished cannot tell whether a retry or the poll answered for it, so it +// deletes nothing, takes its mark back off, and says which read failed. +func TestTheRollbackDeletesNothingWhenItCannotJudgeFinished(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedRolledBackDefinition(t, backend, "dst-unjudged") + + srv := &Server{Store: faultyStore{backend, storeFaults{target: "dst-unjudged", volumeList: errStoreFault}}} + + err := srv.rollBackCompensating(t.Context(), "dst-unjudged", []string{"node-a"}, rollbackUnlessPlaced) + if rollbackStepName(err) != "read-finished" { + t.Errorf("rollback = %v, want it to stop at read-finished", err) + } + + rd, getErr := backend.ResourceDefinitions().Get(t.Context(), "dst-unjudged") + if getErr != nil { + t.Fatalf("the rollback deleted a definition it could not judge: %v", getErr) + } + + if mark := rd.Props[store.RollbackAbandonedProp]; mark != "" { + t.Errorf("the definition left in place carries rollback mark %q", mark) + } + + if cause, _ := rollbackFailureAdvice(err, "dst-unjudged"); !strings.Contains(cause, "volumes and replicas") { + t.Errorf("cause %q does not name the read that failed", cause) + } +} + +// markLandsThenClearsFail lets the rollback's mark through and fails every +// definition patch after it, the clear among them. +type markLandsThenClearsFail struct { + store.Store + + patches *atomic.Int32 +} + +type markLandsThenClearsFailRDs struct { + store.ResourceDefinitionStore + + patches *atomic.Int32 +} + +func (m markLandsThenClearsFail) ResourceDefinitions() store.ResourceDefinitionStore { + return markLandsThenClearsFailRDs{ResourceDefinitionStore: m.Store.ResourceDefinitions(), patches: m.patches} +} + +func (r markLandsThenClearsFailRDs) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + if r.patches.Add(1) > 1 { + return errPatchRefused + } + + return r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) //nolint:wrapcheck // pass-through test double +} + +// A rollback that leaves a finished clone whole and cannot take its mark back +// off leaves one every retry refuses, over a volume the driver was told +// exists. The answer says so and names the command that clears the mark, and +// that command goes through: deleting the clone is not the only way out. +func TestARollbackThatLeftAFinishedCloneWithItsMarkNamesTheWayOut(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-stuck") + + base, stop := startServerWithStore(t, backend) + defer stop() + + if code := cloneOnce(t, base, "src-stuck", "dst-stuck", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + srv := &Server{Store: markLandsThenClearsFail{Store: backend, patches: &atomic.Int32{}}} + + err := srv.rollBackCompensating(ctx, "dst-stuck", nil, rollbackUnlessPlaced) + if !errors.Is(err, errRollbackAnswered) { + t.Fatalf("rollback = %v, want it to leave the finished clone", err) + } + + cause, correc := rollbackFailureAdvice(err, "dst-stuck") + if !strings.Contains(cause, "every retry is refused") || !strings.Contains(correc, "set-property dst-stuck "+store.RollbackAbandonedProp) { + t.Errorf("advice %q / %q does not say the mark stays and how to clear it", cause, correc) + } + + if code := cloneOnce(t, base, "src-stuck", "dst-stuck", nil); code != http.StatusConflict { + t.Fatalf("retry over the stuck mark = %d, want 409", code) + } + + raw, err := json.Marshal(map[string]any{"delete_props": []string{store.RollbackAbandonedProp}}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + put := httpPut(t, base+"/v1/resource-definitions/dst-stuck", raw) + _ = put.Body.Close() + + if put.StatusCode != http.StatusOK { + t.Fatalf("clearing the mark with rd modify = %d, want 200", put.StatusCode) + } + + if code := cloneOnce(t, base, "src-stuck", "dst-stuck", nil); code != http.StatusCreated { + t.Errorf("retry after clearing the mark = %d, want 201", code) + } +} + +// The in-progress refusal names the clear as the way out only after the +// rollback has had its time, and says what clearing it early does: a retry +// adopts the definition and the rollback, still running, deletes it under it. +func TestTheInProgressRefusalNamesTheClearAndWhenItIsSafe(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-inprog", Props: map[string]string{store.RollbackAbandonedProp: store.RollbackInProgress}, + }); err != nil { + t.Fatalf("seed: %v", err) + } + + _, refusal := (&Server{Store: backend}).abandonedRollbackRefusal(ctx, "clone", "dst-inprog") + if refusal == nil { + t.Fatal("no refusal over an in-progress mark") + } + + for _, want := range []string{ + "set-property dst-inprog " + store.RollbackAbandonedProp, "a minute after the failure", "stops that rollback before it deletes anything", + } { + if !strings.Contains(refusal.Correc, want) { + t.Errorf("correction %q does not say %q", refusal.Correc, want) + } + } +} + +// The RG-deleted doors put the group first in their correction; a mark the +// rollback could not take off still refuses the retry after the group is +// back, so clearing it is named between the two. +func TestTheRGDeletedAdviceForAStuckMarkNamesTheClear(t *testing.T) { + t.Parallel() + + _, correc := rollbackFailureAdviceOverDeletedGroup( + &rollbackMarkStuckError{why: errRollbackYielded, err: errPatchRefused}, "dst-rg", "rg-gone") + + for _, want := range []string{"re-create resource group 'rg-gone'", "set-property dst-rg"} { + if !strings.Contains(correc, want) { + t.Errorf("correction %q does not say %q", correc, want) + } + } +} + +// A finished but short restore whose mark could not be taken off names both +// ways out: the clear, and the replicas the retry will not place. +func TestAStuckMarkOnAShortRestoreNamesTheMissingReplicasToo(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-short2") + + base, stop := startServerWithStore(t, backend) + defer stop() + + if code := cloneOnce(t, base, "src-short2", "dst-short2", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + srv := &Server{Store: markLandsThenClearsFail{Store: backend, patches: &atomic.Int32{}}} + + rc := srv.failedMaterialiseRefusal(ctx, "restore failed", "restore", "dst-short2", nil, + rollbackUnlessPlaced, []string{"node-a", "node-zzz"}, &materialiseAfterCreateError{err: errPatchRefused}) + + for _, want := range []string{"set-property dst-short2", "no replica on node-zzz"} { + if !strings.Contains(rc.Correc, want) { + t.Errorf("correction %q does not say %q", rc.Correc, want) + } + } +} + +// Only an in-progress mark is the operator's to clear. One naming the step a +// rollback gave up at sits on a definition the rollback took partly apart, +// and clearing it hands the remains to a retry that resumes over them. +func TestAStepMarkIsNotTheOperatorsToClear(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-step", Props: map[string]string{store.RollbackAbandonedProp: "reap-replicas"}, + }); err != nil { + t.Fatalf("seed: %v", err) + } + + base, stop := startServerWithStore(t, backend) + defer stop() + + raw, err := json.Marshal(map[string]any{"delete_props": []string{store.RollbackAbandonedProp}}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + put := httpPut(t, base+"/v1/resource-definitions/dst-step", raw) + _ = put.Body.Close() + + if put.StatusCode != http.StatusBadRequest { + t.Errorf("clearing a step mark = %d, want 400", put.StatusCode) + } + + rd, err := backend.ResourceDefinitions().Get(ctx, "dst-step") + if err != nil || rd.Props[store.RollbackAbandonedProp] != "reap-replicas" { + t.Errorf("the refused clear changed the mark: %v (%v)", rd.Props, err) + } +} + +// A rollback that stopped at the definition's snapshots touched nothing: the +// definition is whole, and `rd d` refuses it over those snapshots, so +// clearing the mark is the operator's way to keep it without destroying the +// snapshots the rollback refused to destroy. +func TestAMarkOverAWholeDefinitionIsTheOperatorsToClear(t *testing.T) { + t.Parallel() + + for _, step := range []string{store.RollbackStepSnapshots, store.RollbackStepReadSnapshots} { + t.Run(step, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-whole", Props: map[string]string{store.RollbackAbandonedProp: step}, + }); err != nil { + t.Fatalf("seed: %v", err) + } + + base, stop := startServerWithStore(t, backend) + defer stop() + + raw, err := json.Marshal(map[string]any{"delete_props": []string{store.RollbackAbandonedProp}}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + put := httpPut(t, base+"/v1/resource-definitions/dst-whole", raw) + _ = put.Body.Close() + + if put.StatusCode != http.StatusOK { + t.Errorf("clearing a %s mark = %d, want 200", step, put.StatusCode) + } + + if _, cause := rollbackStepAdvice(mustStep(t, step), true, "dst-whole"); !strings.Contains(cause, "set-property dst-whole") { + t.Errorf("advice for a %s mark %q does not name the clear", step, cause) + } + }) + } +} + +func mustStep(t *testing.T, spelled string) rollbackStep { + t.Helper() + + step, known := rollbackStepByName(spelled) + if !known { + t.Fatalf("no rollback step spelled %q", spelled) + } + + return step +} + +// The rule refuses one edit of a definition a rollback gave up on, the delete +// of its step mark, and lets every other through: linstor-csi's own +// delete_props, an operator's Aux edit, the empty-value spelling of the clear +// on a mark over a whole definition. +func TestOtherEditsOfAStepMarkedDefinitionGoThrough(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + mark string + body map[string]any + gone bool + }{ + {name: "aux-delete", mark: "reap-replicas", body: map[string]any{"delete_props": []string{"Aux/keep"}}}, + {name: "aux-set", mark: "reap-replicas", body: map[string]any{"override_props": map[string]string{"Aux/new": "x"}}}, + { + name: "empty-override-clears-in-progress", mark: store.RollbackInProgress, gone: true, + body: map[string]any{"override_props": map[string]string{store.RollbackAbandonedProp: ""}}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-other", + Props: map[string]string{store.RollbackAbandonedProp: tc.mark, "Aux/keep": "y"}, + }); err != nil { + t.Fatalf("seed: %v", err) + } + + base, stop := startServerWithStore(t, backend) + defer stop() + + raw, err := json.Marshal(tc.body) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + put := httpPut(t, base+"/v1/resource-definitions/dst-other", raw) + _ = put.Body.Close() + + if put.StatusCode != http.StatusOK { + t.Fatalf("modify = %d, want 200", put.StatusCode) + } + + rd, err := backend.ResourceDefinitions().Get(ctx, "dst-other") + if err != nil { + t.Fatalf("read back: %v", err) + } + + if _, kept := rd.Props[store.RollbackAbandonedProp]; kept == tc.gone { + t.Errorf("mark kept=%v after %v, want kept=%v", kept, tc.body, !tc.gone) + } + }) + } +} + +// The restore door's abandoned-rollback refusal carries the exists band only +// when it says what stands under the name: a 409 over the mark. A read that +// failed says nothing about it, and its 500 carries no band. +func TestTheRestoreAbandonedRollbackBandOnlyOnAConflict(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + blind bool + wantCode int + wantBand bool + }{ + {name: "mark", wantCode: http.StatusConflict, wantBand: true}, + {name: "read-failed", blind: true, wantCode: http.StatusInternalServerError}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-band2", Props: map[string]string{store.RollbackAbandonedProp: store.RollbackInProgress}, + }); err != nil { + t.Fatalf("seed: %v", err) + } + + var st store.Store = backend + if tc.blind { + st = handshakeBlind{Store: backend, failLiveRead: true} + } + + w := httptest.NewRecorder() + if !(&Server{Store: st}).restoreRollbackWasAbandoned(ctx, w, "dst-band2") { + t.Fatal("no refusal written") + } + + if w.Code != tc.wantCode { + t.Fatalf("refusal = %d, want %d", w.Code, tc.wantCode) + } + + var rcs []apiv1.APICallRc + if err := json.Unmarshal(w.Body.Bytes(), &rcs); err != nil || len(rcs) != 1 { + t.Fatalf("decode %q: %v", w.Body.String(), err) + } + + if got := rcs[0].RetCode&apiCallRcFailExistsRscDfn == apiCallRcFailExistsRscDfn; got != tc.wantBand { + t.Errorf("ret_code %#x carries the exists band = %v, want %v", rcs[0].RetCode, got, tc.wantBand) + } + }) + } +} diff --git a/pkg/rest/rollback_identity_test.go b/pkg/rest/rollback_identity_test.go new file mode 100644 index 00000000..6adf8942 --- /dev/null +++ b/pkg/rest/rollback_identity_test.go @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "context" + "errors" + "fmt" + "strings" + "sync/atomic" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// rdPatchHook runs before the n-th spec patch of target, counting from one, +// and may answer it in place of the store. +type rdPatchHook struct { + store.ResourceDefinitionStore + + target string + calls *atomic.Int32 + hook func(n int32) (error, bool) +} + +func (h rdPatchHook) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + if name == h.target { + if err, answered := h.hook(h.calls.Add(1)); answered { + return err + } + } + + return h.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) //nolint:wrapcheck // test double +} + +type rdPatchHookStore struct { + store.Store + + target string + calls *atomic.Int32 + hook func(n int32) (error, bool) +} + +func (s rdPatchHookStore) ResourceDefinitions() store.ResourceDefinitionStore { + return rdPatchHook{ResourceDefinitionStore: s.Store.ResourceDefinitions(), target: s.target, calls: s.calls, hook: s.hook} +} + +func seedDefinitionWithReplica(t *testing.T, st store.Store, name string, props map[string]string) { + t.Helper() + + ctx := t.Context() + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: name, Props: props}); err != nil { + t.Fatalf("seed %s: %v", name, err) + } + + if err := st.VolumeDefinitions().Create(ctx, name, &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 1024}); err != nil { + t.Fatalf("seed the volume of %s: %v", name, err) + } + + if err := st.Resources().Create(ctx, &apiv1.Resource{Name: name, NodeName: "node-a"}); err != nil { + t.Fatalf("seed the replica of %s: %v", name, err) + } +} + +func assertDefinitionAndReplicaStand(t *testing.T, st store.Store, name string) { + t.Helper() + + if _, err := st.ResourceDefinitions().Get(t.Context(), name); err != nil { + t.Errorf("the definition under %s was deleted: %v", name, err) + } + + if _, err := st.Resources().Get(t.Context(), name, "node-a"); err != nil { + t.Errorf("the replica of %s was deleted: %v", name, err) + } +} + +// The definition a rollback was for is gone by the time it marks it, and +// another writer has a definition under the same name. The rollback has +// nothing of its own left to remove, and reports so without deleting by name. +func TestARollbackWhoseDefinitionIsGoneDeletesNothingByName(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDefinitionWithReplica(t, backend, "dst-gone", nil) + + var calls atomic.Int32 + + st := rdPatchHookStore{Store: backend, target: "dst-gone", calls: &calls, hook: func(n int32) (error, bool) { + if n == 1 { + return store.ErrNotFound, true + } + + return nil, false + }} + + err := (&Server{Store: st}).rollBackCompensating(t.Context(), "dst-gone", []string{"node-a"}, rollbackEvenIfFinished) + if err != nil { + t.Errorf("rollback over a definition already gone = %v, want nil", err) + } + + assertDefinitionAndReplicaStand(t, backend, "dst-gone") +} + +// The definition under the name stops carrying the rollback's own in-progress +// mark between the handshake and the first delete: it was recreated, or an +// operator cleared the mark. Nothing is deleted by name, and the mark is not +// rewritten onto a definition that is not this rollback's. +func TestARollbackDeletesNothingOnceItsMarkIsGone(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDefinitionWithReplica(t, backend, "dst-swapped", nil) + + var calls atomic.Int32 + + st := rdPatchHookStore{Store: backend, target: "dst-swapped", calls: &calls, hook: func(n int32) (error, bool) { + if n != 2 { + return nil, false + } + + // Between the handshake and the first destructive step, the + // definition under the name loses the mark. + _ = backend.ResourceDefinitions().PatchResourceDefinitionSpec(context.Background(), "dst-swapped", + func(rd *apiv1.ResourceDefinition) error { + delete(rd.Props, store.RollbackAbandonedProp) + + return nil + }) + + return nil, false + }} + + err := (&Server{Store: st}).rollBackCompensating(t.Context(), "dst-swapped", []string{"node-a"}, rollbackEvenIfFinished) + if !errors.Is(err, errRollbackMarkNotOurs) { + t.Errorf("rollback = %v, want it to stop over a mark that is no longer its own", err) + } + + assertDefinitionAndReplicaStand(t, backend, "dst-swapped") + + rd, err := backend.ResourceDefinitions().Get(t.Context(), "dst-swapped") + if err == nil && rd.Props[store.RollbackAbandonedProp] != "" { + t.Errorf("the rollback wrote %q onto a definition that is not its own", rd.Props[store.RollbackAbandonedProp]) + } +} + +// A rollback whose first destructive step fails, and which then has no budget +// left to say where it stopped, still leaves a mark that names a step: it went +// on before the step ran. An operator cannot clear it, since the definition +// under it may already be part torn, and an in-progress mark is kept for +// definitions left whole. +func TestARollbackThatDiesTakingTheDefinitionApartLeavesAStepMark(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDefinitionWithReplica(t, backend, "dst-torn", nil) + + var calls atomic.Int32 + + var deleting atomic.Bool + + // Every write after the first delete was tried fails, the way an + // exhausted budget fails it. + st := rdPatchHookStore{Store: backend, target: "dst-torn", calls: &calls, hook: func(int32) (error, bool) { + if deleting.Load() { + return context.DeadlineExceeded, true + } + + return nil, false + }} + + srv := &Server{Store: replicaDeleteRefused{Store: st, tried: &deleting}} + + err := srv.rollBackCompensating(t.Context(), "dst-torn", []string{"node-a"}, rollbackEvenIfFinished) + if err == nil { + t.Fatal("a rollback whose replica delete failed reported success") + } + + rd, err := backend.ResourceDefinitions().Get(t.Context(), "dst-torn") + if err != nil { + t.Fatalf("read the definition: %v", err) + } + + mark := rd.Props[store.RollbackAbandonedProp] + if mark != store.RollbackStepReapReplicas { + t.Errorf("rollback mark = %q, want %q", mark, store.RollbackStepReapReplicas) + } + + cleared := map[string]string{} + if !errors.Is(store.RollbackMarkClearRefusal(rd.Props, cleared), store.ErrRollbackStepMarkKept) { + t.Errorf("an operator may clear %q over a definition the rollback started taking apart", mark) + } +} + +// replicaDeleteRefused fails every replica delete and records that one was +// tried. +type replicaDeleteRefused struct { + store.Store + + tried *atomic.Bool +} + +func (r replicaDeleteRefused) Resources() store.ResourceStore { + return replicaDeleteRefusedResources{ResourceStore: r.Store.Resources(), tried: r.tried} +} + +type replicaDeleteRefusedResources struct { + store.ResourceStore + + tried *atomic.Bool +} + +func (r replicaDeleteRefusedResources) Delete(context.Context, string, string) error { + r.tried.Store(true) + + return context.DeadlineExceeded +} + +// A rollback whose mark was replaced does not tell the operator it failed to +// record that it started: it did, and somebody else's definition is there now. +func TestARollbackWhoseMarkWasReplacedSaysSo(t *testing.T) { + t.Parallel() + + cause, _ := rollbackFailureAdvice(newRollbackError(rollbackStepMark, + fmt.Errorf("%q: %w", "dst-replaced", errRollbackMarkNotOurs)), "dst-replaced") + if strings.Contains(cause, "could not record that it started") { + t.Errorf("advice for a replaced mark = %q, want it to say the mark was cleared or replaced", cause) + } + + if !strings.Contains(cause, "cleared or replaced") { + t.Errorf("advice for a replaced mark = %q", cause) + } +} + +// A definition gone by the time the rollback enters its destructive steps is +// left gone: a definition recreated under the name since is somebody else's, +// and the cascade goes by name. +func TestARollbackWhoseDefinitionGoesBeforeItsFirstDeleteDeletesNothing(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDefinitionWithReplica(t, backend, "dst-regone", nil) + + var calls atomic.Int32 + + st := rdPatchHookStore{Store: backend, target: "dst-regone", calls: &calls, hook: func(n int32) (error, bool) { + if n != 2 { + return nil, false + } + + // The step-mark write finds no definition: it went, and another + // writer has already put a new one with a replica under the name. + return store.ErrNotFound, true + }} + + err := (&Server{Store: st}).rollBackCompensating(t.Context(), "dst-regone", []string{"node-a"}, rollbackEvenIfFinished) + if err != nil { + t.Errorf("rollback over a definition that went = %v, want nil", err) + } + + assertDefinitionAndReplicaStand(t, backend, "dst-regone") +} diff --git a/pkg/rest/server.go b/pkg/rest/server.go index 79ec9ea5..955955ec 100644 --- a/pkg/rest/server.go +++ b/pkg/rest/server.go @@ -367,12 +367,12 @@ func isSSEPath(p string) bool { } // gracefulShutdownWindow is how long Shutdown waits for handlers that are -// still running. It is derived from detachedRollbackBudget, which bounds every -// context a handler detaches from its caller (detachedCompensation: the -// post-write group re-read, the rollbacks, rollbackSpawn), so it is the -// longest a handler can still be working after its caller has gone. See that -// constant for the whole chain, down to the termination grace the manifests -// give the pod. +// still running. It is derived from the budgets of every context a handler +// detaches from its caller, so it is the longest a handler can still be +// working after its caller has gone: detachedRollbackBudget for +// detachedCompensation (the post-write group re-read, the rollbacks, +// rollbackSpawn). See detachedRollbackBudget for the whole chain, down to the +// termination grace the manifests give the pod. const gracefulShutdownWindow = detachedRollbackBudget + shutdownMargin // waitAndShutdown blocks until ctx is cancelled or any listener diff --git a/pkg/rest/snapshot_restore.go b/pkg/rest/snapshot_restore.go index 3fc71418..793e4f76 100644 --- a/pkg/rest/snapshot_restore.go +++ b/pkg/rest/snapshot_restore.go @@ -20,6 +20,7 @@ package rest import ( "context" + "maps" "net/http" "slices" "strconv" @@ -113,13 +114,31 @@ func (s *Server) handleSnapshotRestoreVolumeDefinition(w http.ResponseWriter, r return } - _, err = s.Store.ResourceDefinitions().Get(r.Context(), req.ToResource) + target, err := s.Store.ResourceDefinitions().Get(r.Context(), req.ToResource) if err != nil { writeStoreError(w, err) return } + // A target carrying DELETE is refused, as on the sibling handlers: + // hydrating volumes into it races the tear-down reaping what it writes. + if slices.Contains(target.Flags, rdFlagDelete) { + writeJSON(w, http.StatusConflict, []apiv1.APICallRc{{ + RetCode: apiCallRcError | apiCallRcFailExistsRscDfn, + Message: "resource definition '" + req.ToResource + "' is being deleted", + Cause: "the target carries the DELETE flag; the volumes this would hydrate " + + "are being reaped as it writes them", + Correc: "wait for the delete to finish, then restore into a fresh definition", + }}) + + return + } + + if !s.refuseVolumeRestoreIntoMarkedTarget(r.Context(), w, req.ToResource) { + return + } + // G3b (corner-case): the VD-restore variant hydrates the snapshot's // recorded volume layout onto a (typically pre-existing, empty) // target RD. If the target RD ALREADY carries a volume-definition @@ -133,7 +152,7 @@ func (s *Server) handleSnapshotRestoreVolumeDefinition(w http.ResponseWriter, r return } - err = hydrateVolumesFromSnapshot(r.Context(), s, req.ToResource, &snap) + err = hydrateVolumesFromSnapshot(r.Context(), s, req.ToResource, &snap, false) if err != nil { writeStoreError(w, err) @@ -147,6 +166,35 @@ func (s *Server) handleSnapshotRestoreVolumeDefinition(w http.ResponseWriter, r }}) } +// refuseVolumeRestoreIntoMarkedTarget refuses a volume-definition restore into +// a definition another restore or clone has marked as its own, as the CLI +// does: that operation hydrates the volumes itself, and its rollback, if it +// fails, takes the definition with whatever this answered 200 for. The marker +// goes on with the definition, and is read past the cache. False means a +// refusal has been written. +func (s *Server) refuseVolumeRestoreIntoMarkedTarget(ctx context.Context, w http.ResponseWriter, toResource string) bool { + live, err := s.Store.ResourceDefinitions().GetUncached(ctx, toResource) + if err != nil { + writeStoreError(w, err) + + return false + } + + marker := live.Props[restoreFromSnapshotKey] + if marker == "" { + return true + } + + writeJSON(w, http.StatusConflict, []apiv1.APICallRc{{ + RetCode: apiCallRcError, + Message: "resource definition '" + toResource + "' is the target of a restore or clone", + Cause: "it carries the restore marker '" + marker + "', and that operation writes its volumes itself", + Correc: "restore it with `linstor snapshot resource restore` instead", + }}) + + return false +} + // refuseRestoreOnVolumeConflict is the G3b pre-mutation guard for the // VD-restore handler. It compares the snapshot's recorded volume // numbers against the target RD's existing VolumeDefinitions and @@ -255,6 +303,10 @@ func (s *Server) handleSnapshotRestore(w http.ResponseWriter, r *http.Request) { return } + if !s.rebindRestoreTarget(r.Context(), w, &req) { + return + } + snapName := resolveSnapshotName(r, &req) if snapName == "" { writeError(w, http.StatusBadRequest, "snapshot name required (URL path, from_snapshot, or snapshot_name)") @@ -266,10 +318,8 @@ func (s *Server) handleSnapshotRestore(w http.ResponseWriter, r *http.Request) { // snapshot POSTs the restore right after the snapshot create; // absorb informer-cache lag on the source-snapshot read instead // of 404-ing the restore. - snap, err := getSnapshotWithCacheRetry(r.Context(), s.Store, srcRD, snapName) - if err != nil { - writeStoreError(w, err) - + snap, ok := s.restoreSnapshotOrReplay(r.Context(), w, srcRD, snapName, &req) + if !ok { return } @@ -315,9 +365,186 @@ func (s *Server) handleSnapshotRestore(w http.ResponseWriter, r *http.Request) { // target is left an empty shell for the operator / linstor-csi to // place (restore-then-scale-out); an explicit node list is still // stamped verbatim inside materializeRestoredRD. - made, err := s.materializeRestoredRD(r.Context(), srcRD, &req, &snap, false) + // A leftover from an earlier attempt is resumed, not reported done: + // the marker is stamped with the definition, before its volumes and + // replicas exist, so on its own it says a restore started, not that it + // finished. + prepared, stop := s.adoptPreparedRestoreTarget(r.Context(), w, &snap, req.ToResource) + if stop { + return + } + + resume, stop := s.restoreReplayState(r.Context(), w, &snap, &req, prepared) + if stop { + return + } + + // A target the caller prepared is this restore's first run, not a retry. + s.materializeRestore(r.Context(), w, srcRD, &req, &snap, resume && !prepared) +} + +// rebindRestoreTarget names the restore's target as it is stored; see +// storedDefinitionName. False means a refusal has been written. +func (s *Server) rebindRestoreTarget(ctx context.Context, w http.ResponseWriter, req *snapshotRestoreRequest) bool { + stored, err := s.storedDefinitionName(ctx, req.ToResource) + if err != nil { + writeRestoreTargetUnreadable(w, req.ToResource, err) + + return false + } + + req.ToResource = stored + + return true +} + +// writeRestoreTargetUnreadable refuses a restore whose target could not be +// read. Read as absent, the create that follows collides with what is there +// and is adopted past every judgement of a leftover (finished, torn down, +// parented to a group that is gone), so a replay of a finished restore +// re-placed it on a node the operator had emptied; the retry this answer asks +// for gets the judgement. +func writeRestoreTargetUnreadable(w http.ResponseWriter, toResource string, err error) { + writeJSON(w, http.StatusInternalServerError, []apiv1.APICallRc{{ + RetCode: apiCallRcError, + Message: "restore target '" + toResource + "' could not be read: " + scrubImplDetails(err.Error()), + Correc: "retry the restore", + }}) +} + +// adoptPreparedRestoreTarget takes a definition the caller prepared for this +// restore as its target: created empty, its volume definitions restored from +// this snapshot, no replica yet. That is how LINSTOR's own restore sequence +// runs, and how linstor-csi restores every volume from a snapshot; see +// store.PreparedRestoreTarget. The marker is stamped on it here, so from then +// on the target reads as this restore's own, its first run and every retry +// alike. It returns (adopted, stop). +func (s *Server) adoptPreparedRestoreTarget( + ctx context.Context, w http.ResponseWriter, snap *apiv1.Snapshot, toResource string, +) (bool, bool) { + // Past the cache, like every read this path decides on: the caller + // created this definition one call earlier, possibly through another + // apiserver replica, and a cache that holds an older copy under the name + // (deleted and recreated, or patched since) answers the marker wrong. + existing, err := s.Store.ResourceDefinitions().GetUncached(ctx, toResource) + if errors.Is(err, store.ErrNotFound) { + return false, false + } + + if err != nil { + writeRestoreTargetUnreadable(w, toResource, err) + + return false, true + } + + prepared, err := store.PreparedRestoreTarget(ctx, s.Store, &existing, snap) + if err != nil { + writePreparedTargetRefusal(w, existing.Name, snap, err) + + return false, true + } + + if !prepared { + return false, false + } + + // Everything that can still refuse this target runs before the marker + // goes on it: the definition is the caller's, and a refusal after the + // stamp would leave it looking like a restore that started. + if !s.restoreTargetGroupSurvived(ctx, w, &existing) { + return false, true + } + + adopted, err := store.AdoptPreparedRestoreTarget(ctx, s.Store, existing.Name, snap) + if errors.Is(err, store.ErrRestoreTargetTaken) { + // Another restore marked it first; restoreTargetState refuses it + // as somebody else's, with the wording that already says why. + return false, false + } + + if err != nil { + writePreparedTargetRefusal(w, existing.Name, snap, err) + + return false, true + } + + // Not adopted with no error: another request for this same restore + // marked it first, so it is that request's leftover, judged by the + // replay gate like any other. + return adopted, false +} + +// writePreparedTargetRefusal answers a target that is prepared in every +// respect but one that makes restoring into it wrong; see +// store.PreparedRestoreTarget. Anything else is the store error it is. +func writePreparedTargetRefusal(w http.ResponseWriter, rdName string, snap *apiv1.Snapshot, err error) { + switch { + case errors.Is(err, store.ErrRestoreTargetLayers): + writeJSON(w, http.StatusConflict, []apiv1.APICallRc{{ + RetCode: apiCallRcError | apiCallRcFailExistsRscDfn, + Message: "resource definition '" + rdName + "' cannot take the restore of '" + + snap.Name + "': " + scrubImplDetails(err.Error()), + Cause: "the restore puts the source's data under the target's layer stack, so a " + + "layer the source did not have writes its own metadata across it, and one it " + + "had is missing on read", + Correc: "create '" + rdName + "' with the layer stack of '" + snap.ResourceName + + "' (for linstor-csi, a StorageClass with the same layer list), or restore under " + + "a name that does not exist yet", + }}) + case errors.Is(err, store.ErrRestoreTargetTearingDown): + writeJSON(w, http.StatusConflict, []apiv1.APICallRc{{ + RetCode: apiCallRcError | apiCallRcFailExistsRscDfn, + Message: "resource definition '" + rdName + "' still has replicas being deleted", + Cause: "placing the restore over them would race the tear-down reaping them", + Correc: "wait for the replicas of '" + rdName + "' to go, then re-issue the restore", + }}) + default: + writeStoreError(w, err) + } +} + +// restoreTargetGroupSurvived refuses an existing target parented to a group +// that is gone, before the restore writes anything into it: a prepared target +// before it is marked, a leftover before it is resumed. The post-write guard +// would otherwise refuse the same target after the write, and it does not +// roll back a definition this request did not create. +func (s *Server) restoreTargetGroupSurvived( + ctx context.Context, w http.ResponseWriter, existing *apiv1.ResourceDefinition, +) bool { + survived, err := s.parentRGSurvived(ctx, existing.ResourceGroupName) + if err != nil { + writeJSON(w, http.StatusInternalServerError, []apiv1.APICallRc{{ + RetCode: apiCallRcError, + Message: "resource definition '" + existing.Name + "' exists, but its resource group '" + + existing.ResourceGroupName + "' could not be read: " + scrubImplDetails(err.Error()), + Correc: "retry the restore", + }}) + + return false + } + + if survived { + return true + } + + writeJSON(w, http.StatusConflict, []apiv1.APICallRc{ + *adoptedOverDeletedGroupRefusal("restore", existing.Name, existing.ResourceGroupName, + correcRecreateGroupThenRestore), + }) + + return false +} + +// materializeRestore restores the target, guards the group it was written +// with and answers. resume says whether this request is finishing a leftover, +// which only the wording sees. +func (s *Server) materializeRestore( + ctx context.Context, w http.ResponseWriter, srcRD string, req *snapshotRestoreRequest, + snap *apiv1.Snapshot, resume bool, +) { + made, err := s.materializeRestoredRD(ctx, srcRD, req, snap, false, nil) if err != nil { - s.writeRestoreMaterialiseFailed(r.Context(), w, snapName, req.ToResource, made, err) + s.writeRestoreMaterialiseFailed(ctx, w, snap.Name, req, made, err) return } @@ -325,34 +552,346 @@ func (s *Server) handleSnapshotRestore(w http.ResponseWriter, r *http.Request) { // The group validated is the one that was WRITTEN, not the source's read // back a second time: re-reading answers a different question, and the // extra read was itself a way to fail a restore that had already worked. - uncheckedRG, ok := s.restoreParentRGSurvived(r.Context(), w, made) + uncheckedRG, ok := s.restoreParentRGSurvived(ctx, w, made) if !ok { return } - writeRestoreDone(w, "snapshot restored: "+snapName+" → "+made.Name, uncheckedRG) + writeRestoreDone(w, restoreDoneMessage(resume, snap.Name, made.Name), uncheckedRG) } // writeRestoreMaterialiseFailed answers a restore whose materialisation failed. // -// A failure after the create is this request's own partial work, and this -// endpoint has no idempotent-replay gate: left in place, the marker-bearing -// definition turns every retry under the deterministic CSI target name into -// AlreadyExists, for good. So that work is rolled back before the answer goes -// out. Anything else is answered as the store error it is. +// A failure after this request's own create is its own partial work, and it +// is rolled back before the answer goes out: the retry under the +// deterministic CSI target name then starts clean rather than resuming over +// half of a definition. A leftover this request adopted is never wrapped as +// that, so it stays for the replay gate to judge. Anything else is answered as +// the store error it is. func (s *Server) writeRestoreMaterialiseFailed( - ctx context.Context, w http.ResponseWriter, snapName, rdName string, made materialisedRD, err error, + ctx context.Context, w http.ResponseWriter, snapName string, req *snapshotRestoreRequest, + made materialisedRD, err error, ) { - var partial *materialiseAfterCreateError - if !errors.As(err, &partial) { - writeStoreError(w, err) + rdName := req.ToResource + + // Spared on the terms the replay judges a finished restore by: with a + // replica when this request placed some, by its volumes alone when not. + wanted := canonicalRestoreNodeList(req) + + scope := rollbackUnlessHydrated + if len(wanted) > 0 { + scope = rollbackUnlessPlaced + } + + if kind, refused := materialiseRefusalKind(err); refused { + writeStoreErrorTyped(w, err, kind) return } writeJSON(w, http.StatusInternalServerError, []apiv1.APICallRc{*s.failedMaterialiseRefusal(ctx, "snapshot restore of '"+snapName+"' into '"+rdName+"' failed: "+err.Error(), - "restore", rdName, made.Placed, err)}) + "restore", rdName, made.Placed, scope, wanted, err)}) +} + +// materialiseRefusalKind sorts a failed materialisation that is not this +// request's own partial work: an answer about what was already there (a +// replica still being deleted, a creator already rolling back), which both +// doors give in the same typed shape. It reports false for partial work, which +// the caller undoes. +// +// A replica still being deleted, or one holding no data, carries no +// FAIL_EXISTS band: the replica it names is not one that exists for the +// caller to use, and the 409 is the retryable failure it is. +func materialiseRefusalKind(err error) (storeResourceKind, bool) { + var partial *materialiseAfterCreateError + if errors.As(err, &partial) { + return storeKindUnknown, false + } + + switch { + case errors.Is(err, errReplicaStillDeleting): + return storeKindUnknown, true + case errors.Is(err, errAdoptedLeftoverRollingBack): + return storeKindResourceDfn, true + default: + return storeKindUnknown, true + } +} + +// restoreReplayState is restoreTargetState followed, over a leftover of this +// restore, by the finished question. It returns (resume, stop): stop when an +// answer has been written, the replay of a finished restore included. +// +// prepared says the target is one the caller prepared and this request just +// marked: its first run, not a retry, which only the wording sees. +// +// The gates over an unfinished leftover run before the resume writes into it, +// in the order the clone door runs them: tear-down, then the group, then the +// abandoned-rollback mark. A resume over a definition whose group is gone +// would hydrate and place into it and only then be refused by the group guard +// after the write, which does not roll back what it adopted. +func (s *Server) restoreReplayState( + ctx context.Context, w http.ResponseWriter, snap *apiv1.Snapshot, req *snapshotRestoreRequest, prepared bool, +) (bool, bool) { + resume, stop := s.restoreTargetState(ctx, w, snap, req.ToResource) + if stop || !resume { + return resume, stop + } + + finished, halt := s.restoreLeftoverIsFinished(ctx, w, snap, snap.Name, req) + if halt { + return false, true + } + + if finished { + s.writeFinishedRestoreReplay(ctx, w, snap.Name, req.ToResource, !prepared) + + return false, true + } + + existing, err := s.Store.ResourceDefinitions().Get(ctx, req.ToResource) + if err != nil { + writeStoreError(w, err) + + return false, true + } + + if !s.restoreTargetGroupSurvived(ctx, w, &existing) { + return false, true + } + + if s.restoreRollbackWasAbandoned(ctx, w, req.ToResource) { + return false, true + } + + return true, false +} + +// writeFinishedRestoreReplay answers the replay of a restore that finished. +// The leftover is adopted, not this request's, so the group guard over it +// refuses rather than rolls back when its group is gone, the same answer the +// resume of an unfinished one gets. +func (s *Server) writeFinishedRestoreReplay( + ctx context.Context, w http.ResponseWriter, snapName, rdName string, resumed bool, +) { + existing, err := s.Store.ResourceDefinitions().Get(ctx, rdName) + if err != nil { + writeStoreError(w, err) + + return + } + + // The pre-write gate, not the post-write one: the replay answers for a + // definition nobody verified, so a group it cannot read refuses, as the + // clone's replay and the CLI restore refuse it, rather than answering 201 + // with a warning that binds a PV to a definition parented to nothing. + if !s.restoreTargetGroupSurvived(ctx, w, &existing) { + return + } + + if s.restoreRollbackWasAbandoned(ctx, w, existing.Name) { + return + } + + if status, refusal := s.finishedLeftoverRefusal(ctx, "restore", existing.Name); refusal != nil { + if status == http.StatusConflict { + refusal.RetCode |= apiCallRcFailExistsRscDfn + } + + writeJSON(w, status, []apiv1.APICallRc{*refusal}) + + return + } + + writeRestoreDone(w, restoreDoneMessage(resumed, snapName, rdName), nil) +} + +// restoreLeftoverIsFinished answers a retry over a leftover of this restore +// before anything is re-run over it. It returns (finished, stop). +// +// A finished restore is not placed again: re-placing it would put a replica on +// every requested node that no longer has one, including a node the operator +// emptied since, restored from the point-in-time beside a replica that has +// moved on with live writes. It is judged the way a finished clone is, by its +// volumes and, when the request placed replicas, by holding at least one; +// where they are is placement's business. +// +// snap is nil once the snapshot is gone; the leftover is then judged against +// the volumes the definition recorded when it was restored, the way a clone +// whose internal snapshot is gone is. A bare restore is finished without a +// replica only while the snapshot its later placement restores from exists: +// once that is gone too, a shell with no replica holds its data nowhere, and +// it is judged as the CLI judges it, needing a replica. +func (s *Server) restoreLeftoverIsFinished( + ctx context.Context, w http.ResponseWriter, snap *apiv1.Snapshot, snapName string, req *snapshotRestoreRequest, +) (bool, bool) { + vds, err := store.LiveVolumes(ctx, s.Store, req.ToResource) + if err != nil { + writeStoreError(w, err) + + return false, true + } + + needReplica := len(canonicalRestoreNodeList(req)) > 0 || snap == nil + + progress, err := assessLeftover(ctx, s.Store, req.ToResource, vds, snap, needReplica) + if err != nil { + writeStoreError(w, err) + + return false, true + } + + switch progress { + case cloneFinished: + return true, false + case cloneForeign: + // Once the snapshot is gone, restoring again after the delete meets + // nothing to restore from, the tear-down's wording below. + correc := "delete '" + req.ToResource + "' and restore again, or restore under a different name" + if snap == nil { + correc = "delete '" + req.ToResource + "' and restore it from another snapshot, or restore " + + "another snapshot under a different name" + } + + writeJSON(w, http.StatusConflict, []apiv1.APICallRc{{ + RetCode: apiCallRcError | apiCallRcFailExistsRscDfn, + Message: "resource definition '" + req.ToResource + "' holds a volume the restore of '" + + snapName + "' would not have written", + Cause: "the definition under that name carries this restore's marker, but its volumes " + + "are not the ones the snapshot recorded, so resuming would report complete a " + + "layout nothing restored", + Correc: correc, + }}) + + return false, true + case cloneTearingDown: + // Once the snapshot is gone, restoring again after the tear-down + // meets a target with nothing to restore from. + correc := "wait until the replicas of '" + req.ToResource + "' are gone, then restore again" + if snap == nil { + correc = "wait until the replicas of '" + req.ToResource + "' are gone, then delete '" + + req.ToResource + "' and restore it from another snapshot" + } + + writeJSON(w, http.StatusConflict, []apiv1.APICallRc{{ + RetCode: apiCallRcError | apiCallRcFailExistsRscDfn, + Message: "resource definition '" + req.ToResource + "' is still being torn down", + Cause: "the definition under that name carries this restore's marker, and every " + + "replica of it is already accepted for deletion; resuming would race the " + + "tear-down reaping what it writes", + Correc: correc, + }}) + + return false, true + case cloneUnfinished: + } + + return false, false +} + +// restoreSnapshotOrReplay reads the snapshot a restore is taken from. One +// that is gone hands the request to replayWithoutSnapshot, which answers it. +func (s *Server) restoreSnapshotOrReplay( + ctx context.Context, w http.ResponseWriter, srcRD, snapName string, req *snapshotRestoreRequest, +) (apiv1.Snapshot, bool) { + snap, err := getSnapshotWithCacheRetry(ctx, s.Store, srcRD, snapName) + if errors.Is(err, store.ErrNotFound) { + s.replayWithoutSnapshot(ctx, w, srcRD, snapName, req, err) + + return snap, false + } + + if err != nil { + writeStoreError(w, err) + + return snap, false + } + + return snap, true +} + +// replayWithoutSnapshot answers a restore whose snapshot is gone. Deleting +// the snapshot once the restore finished is ordinary cleanup, and a retry +// after it is still a replay: an absent snapshot cannot unmake a finished +// target, so a target carrying this restore's marker and the record of the +// volumes the snapshot held is judged against that record, the way a clone +// whose internal snapshot is gone is. A finished one answers as the replay it +// is; an unfinished one cannot be finished from a snapshot that is gone, and +// the refusal says so. Anything else, a legacy target without the record +// included, is the 404 it was. +func (s *Server) replayWithoutSnapshot( + ctx context.Context, w http.ResponseWriter, srcRD, snapName string, req *snapshotRestoreRequest, notFound error, +) { + existing, err := s.Store.ResourceDefinitions().GetUncached(ctx, req.ToResource) + if err != nil && !errors.Is(err, store.ErrNotFound) { + // Whether the target is a finished replay or an unfinished leftover + // is not known, so the snapshot's 404 would turn a failed read into + // a refusal that reads as permanent. + writeJSON(w, http.StatusInternalServerError, []apiv1.APICallRc{{ + RetCode: apiCallRcError, + Message: "snapshot '" + snapName + "' of '" + srcRD + "' is gone, and '" + req.ToResource + + "' could not be read to tell whether the restore into it finished: " + scrubImplDetails(err.Error()), + Correc: "retry the restore", + }}) + + return + } + + if err != nil || !restoreMarkerMatches(existing.Props, srcRD, snapName) { + writeStoreError(w, notFound) + + return + } + + if _, recorded := store.RecordedRestoreVolumes(existing.Props); !recorded { + writeStoreError(w, notFound) + + return + } + + if slices.Contains(existing.Flags, rdFlagDelete) { + writeJSON(w, http.StatusConflict, []apiv1.APICallRc{{ + RetCode: apiCallRcError | apiCallRcFailExistsRscDfn, + Message: "resource definition '" + req.ToResource + "' is being deleted", + Cause: "the leftover from an earlier attempt at this restore carries the DELETE " + + "flag; finishing it would race the tear-down reaping what it writes", + Correc: "wait for the delete to finish, then re-issue the restore", + }}) + + return + } + + finished, halt := s.restoreLeftoverIsFinished(ctx, w, nil, snapName, req) + if halt { + return + } + + if finished { + s.writeFinishedRestoreReplay(ctx, w, snapName, req.ToResource, true) + + return + } + + writeJSON(w, http.StatusNotFound, []apiv1.APICallRc{{ + RetCode: apiCallRcError, + Message: "snapshot '" + snapName + "' of '" + srcRD + "' is gone, and the restore into '" + + req.ToResource + "' never finished", + Cause: "the definition carries this restore's marker, but not every volume the snapshot " + + "held, and what is missing can only come from the snapshot", + Correc: "delete '" + req.ToResource + "' and restore it from another snapshot", + }}) +} + +// restoreDoneMessage names whether the restore finished a leftover, so a +// retry is legible in the operator's own output rather than looking like a +// first run. +func restoreDoneMessage(resumed bool, snapName, rdName string) string { + if resumed { + return "snapshot restore completed on retry: " + snapName + " → " + rdName + } + + return "snapshot restored: " + snapName + " → " + rdName } // writeRestoreDone emits the restore's success envelope, with any warning the @@ -370,6 +909,127 @@ func writeRestoreDone(w http.ResponseWriter, message string, warn *apiv1.APICall writeJSON(w, http.StatusCreated, rcs) } +// restoreTargetState decides what an existing definition under the target +// name means. It returns (resume, stop): stop when an answer has already been +// written, resume when the caller should re-run the restore over the leftover. +// +// CSI requires CreateVolume to be idempotent: a repeat with the same name and +// the same parameters has to succeed and return the volume that already +// exists. external-provisioner has no other way to make progress after a +// partial failure, so a leftover of this restore is resumed rather than +// refused as already existing. +// +// The restore marker is what makes a leftover recognisable, and it is NOT +// evidence that the restore finished: materializeRestoredRD stamps it with +// the definition and hydrates the volumes and places the replicas afterwards, +// so a failure in either leaves the marker on an empty shell. That is why +// this resumes rather than reporting success — the restore steps tolerate +// objects a previous attempt already created, so re-running them completes +// what is missing and leaves what is there. +// +// A leftover mid-tear-down is refused rather than resumed: the deletion is +// reaping the very objects finishing the restore would be writing. +// +// Anything else under that name is a genuine collision and stays a refusal: a +// name holding somebody else's definition must not come back as a restore +// that never happened. +// +// The snapshot is taken as the stored object, not as the names the request +// spelled, because the marker is written off the same object: LINSTOR folds +// name case, and a retry arriving as `--from-snapshot SNAP` over a marker +// stamped `snap` must still read as this restore's own. +// +// The target is read from the API server, as is the re-read after a create +// meets AlreadyExists: both decide on the marker, which +// adoptPreparedRestoreTarget may have just stamped through the API server, and +// a cache that has not seen that write serves the definition unmarked. +func (s *Server) restoreTargetState(ctx context.Context, w http.ResponseWriter, snap *apiv1.Snapshot, toResource string) (bool, bool) { + existing, err := s.Store.ResourceDefinitions().GetUncached(ctx, toResource) + if errors.Is(err, store.ErrNotFound) { + return false, false + } + + if err != nil { + writeRestoreTargetUnreadable(w, toResource, err) + + return false, true + } + + if !restoreMarkerMatches(existing.Props, snap.ResourceName, snap.Name) { + writeJSON(w, http.StatusConflict, []apiv1.APICallRc{{ + RetCode: apiCallRcError | apiCallRcFailExistsRscDfn, + Message: "resource definition '" + toResource + "' already exists and is not a restore of '" + + snap.Name + "'", + Correc: "restore under a different name, or delete the existing resource definition first", + }}) + + return false, true + } + + if slices.Contains(existing.Flags, rdFlagDelete) { + writeJSON(w, http.StatusConflict, []apiv1.APICallRc{{ + RetCode: apiCallRcError | apiCallRcFailExistsRscDfn, + Message: "resource definition '" + toResource + "' is being deleted", + Cause: "the leftover from an earlier attempt at this restore carries the DELETE " + + "flag; finishing it would race the tear-down reaping what it writes", + Correc: "wait for the delete to finish, then re-issue the restore", + }}) + + return false, true + } + + return true, false +} + +// restoreRollbackWasAbandoned is the abandoned-rollback gate on the restore +// door. The restore writes the mark through the same failed materialisation +// the clone does, so it reads it back the same way and in the same place: after +// the tear-down and group refusals, which are more precise about the same +// leftover. See abandonedRollbackRefusal. +func (s *Server) restoreRollbackWasAbandoned(ctx context.Context, w http.ResponseWriter, rdName string) bool { + status, refusal := s.abandonedRollbackRefusal(ctx, "restore", rdName) + if refusal == nil { + return false + } + + // The exists band says what stands under the name; a read that failed + // says nothing about it. + if status == http.StatusConflict { + refusal.RetCode |= apiCallRcFailExistsRscDfn + } + + writeJSON(w, status, []apiv1.APICallRc{*refusal}) + + return true +} + +// restoreFromSnapshotKey marks a definition as produced by a snapshot +// restore, encoded `:`. The satellite reads it to route +// the storage provider to RestoreVolumeFromSnapshot, and the retry path above +// reads it to tell its own leftover from somebody else's definition. +const restoreFromSnapshotKey = store.RestoreFromSnapshotProp + +// restoreMarker builds that value. Both halves come off the stored Snapshot +// rather than off the request, so the marker a retry compares is the marker +// the first attempt wrote whichever way the caller spelled the names. +func restoreMarker(srcRD, snapName string) string { + return srcRD + ":" + snapName +} + +// restoreMarkerMatches reports whether a definition was produced by this +// restore or clone. +// +// The comparison is case-insensitive because the two sides reach it from +// different places: the marker is written from the stored objects, and a +// caller derives the other side from names it spelled itself. LINSTOR folds +// name case and pkg/store/k8s/crdname.go lowercases every lookup key, so both +// spellings address one object — and a byte comparison here would answer that +// somebody else owns a definition this restore created, which is the +// terminal-on-first-failure behaviour the resume path exists to end. +func restoreMarkerMatches(props map[string]string, srcRD, snapName string) bool { + return strings.EqualFold(props[restoreFromSnapshotKey], restoreMarker(srcRD, snapName)) +} + // validateRestoreNodesHoldSnapshot is the Bug 397 input-validation guard // for the explicit `--node-name` restore path. It rejects the request when // any requested node does NOT appear in the snapshot's node set @@ -431,6 +1091,73 @@ func resolveSnapshotName(r *http.Request, req *snapshotRestoreRequest) string { return req.SnapshotName } +// leftoverIsThisRestore is the AlreadyExists tolerance's whole question: the +// marker says the definition is this operation's own, the DELETE flag says +// whether it is still there to finish, and the fields the caller named say +// whether it is the same operation. +func leftoverIsThisRestore(existing *apiv1.ResourceDefinition, snap *apiv1.Snapshot, overrides *rdShapeOverrides) bool { + var namedRG string + + var namedLayers []string + + if overrides != nil { + namedRG, namedLayers = overrides.ResourceGroupName, overrides.LayerStack + } + + return restoreMarkerMatches(existing.Props, snap.ResourceName, snap.Name) && + !slices.Contains(existing.Flags, rdFlagDelete) && + requestedShapeDiffers(existing, namedRG, namedLayers) == "" +} + +// requestedShapeDiffers names the first way a definition already under the +// target name differs from what THIS request asked for, or "" when it asked for +// nothing the leftover does not already have. +// +// A retry that resumes a leftover keeps the leftover, so a request naming a +// different resource_group or layer stack would get that shape validated and +// then dropped while the answer says the operation completed — the +// accept-and-drop this endpoint refuses external_name and volume_passphrases to +// avoid. The parent group is not cosmetic: it decides replica count and pool +// selection. +// +// Only the fields the caller NAMED are compared, and only against the +// leftover, never against the live source: the source can change between +// attempts (an ordinary `rd modify --resource-group`), and comparing against +// it would turn every later retry into a permanent 409 with a correction +// linstor-csi cannot act on, since it sends the same body every time. A retry +// that names nothing resumes what the first attempt started, whatever the +// source has become, and one that names what the first attempt stamped, which +// is what linstor-csi's does, resumes it too. +// +// The empty-name guard on the group is load-bearing: the leftover carries the +// group materializeRestoredRD stamped, the source's unless the request named +// one, so without it a request that names no group is compared as "" against +// that group and refused. +func requestedShapeDiffers(existing *apiv1.ResourceDefinition, rgName string, layers []string) string { + if rgName != "" && !strings.EqualFold(existing.ResourceGroupName, rgName) { + return "resource group '" + existing.ResourceGroupName + "', not '" + rgName + "'" + } + + if len(layers) == 0 { + return "" + } + + // An unset stack is a definition that never said, which is what + // materializeRestoredRD copies off a source that never said either, and + // not a definition with no layers. Unresolved, that leftover compared + // against a retry naming [DRBD, STORAGE], which is every linstor-csi + // retry, reads as adding both layers and refuses the resume. A retry that + // names no stack is not compared at all. + have := store.EffectiveLayerStack(existing.LayerStack) + + added, dropped := layerSetDifference(have, layers) + if len(added) > 0 || len(dropped) > 0 { + return "layer stack " + strings.Join(have, ",") + ", not " + strings.Join(layers, ",") + } + + return "" +} + // uncheckedRestoreGroupWarning tells the caller the restore worked and that // the group behind it went unverified, which is the one piece of information // that would make them look. @@ -438,7 +1165,7 @@ func uncheckedRestoreGroupWarning(rdName, rgName string, err error) *apiv1.APICa return &apiv1.APICallRc{ RetCode: maskWarn, Message: "resource group '" + rgName + "' could not be re-checked after the " + - "restore: " + err.Error(), + "restore: " + scrubImplDetails(err.Error()), Cause: "the restore itself succeeded; only the safety net over it could not be " + "inspected, so a group deleted during the restore would not have been caught", Correc: "confirm resource group '" + rgName + "' still exists", @@ -466,10 +1193,7 @@ func (s *Server) restoreParentRGSurvived( // The CHECK failed, which says nothing about the restore: that // already succeeded, and this is the safety net over it. Undoing a // completed restore because the net could not be inspected trades a - // rare dangling group for a certain lost restore — and a much worse - // one, since this endpoint has no idempotent-replay gate, so the - // definition stays behind and every retry under that name meets - // AlreadyExists and answers 409 from then on. + // rare dangling group for a certain lost restore. // // getRGWithCacheRetry returns immediately on anything that is not // NotFound, so this branch is apiserver unavailability, a timeout or @@ -497,9 +1221,9 @@ func (s *Server) restoreParentRGSurvived( return nil, false } - rollbackErr := s.rollBackCompensating(ctx, newRDName, made.Placed) + rollbackErr := s.rollBackCompensating(ctx, newRDName, made.Placed, rollbackEvenIfFinished) if rollbackErr != nil { - cause, correc := rollbackFailureAdvice(rollbackErr, newRDName) + cause, correc := rollbackFailureAdviceOverDeletedGroup(rollbackErr, newRDName, stampedRG) writeJSON(w, http.StatusInternalServerError, []apiv1.APICallRc{{ RetCode: apiCallRcError, @@ -550,7 +1274,56 @@ func (s *Server) restoreParentRGSurvived( // // An explicit caller node list is always stamped verbatim, regardless of // eagerPlace. -func (s *Server) materializeRestoredRD(ctx context.Context, srcRD string, req *snapshotRestoreRequest, snap *apiv1.Snapshot, eagerPlace bool) (materialisedRD, error) { +// rdShapeOverrides carries the parts of a definition's shape a caller may +// choose for itself rather than inherit from the source. Nil means "inherit +// everything", which is what a snapshot restore does. +type rdShapeOverrides struct { + // LayerStack replaces the source's stack when non-empty. + LayerStack []string + // ResourceGroupName replaces the source's parent group when non-empty. + ResourceGroupName string + // OverrideProps, DeleteProps and DeleteNamespaces are the clone's prop + // edits, folded in before the definition is created. Applied only once + // the clone finished, every leftover a failed one kept carried what the + // caller asked to remove: linstor-csi deletes the source's + // Aux/csi-provisioning-completed-by on every clone, and its CreateVolume + // takes a definition that carries it, with volumes of the right size, + // as one already provisioned, so its retry never reaches the clone that + // would resume it. + OverrideProps map[string]string + DeleteProps []string + DeleteNamespaces []string +} + +// applyTo puts a clone's own group, stack and prop edits on the definition it +// is about to create; a restore passes none and inherits the source's. +func (o *rdShapeOverrides) applyTo(rd *apiv1.ResourceDefinition) { + if o == nil { + return + } + + if len(o.LayerStack) > 0 { + rd.LayerStack = o.LayerStack + } + + if o.ResourceGroupName != "" { + rd.ResourceGroupName = o.ResourceGroupName + } + + if rd.Props == nil { + rd.Props = make(map[string]string, len(o.OverrideProps)) + } + + maps.Copy(rd.Props, o.OverrideProps) + + for _, k := range o.DeleteProps { + delete(rd.Props, k) + } + + deletePropNamespaces(rd.Props, o.DeleteNamespaces) +} + +func (s *Server) materializeRestoredRD(ctx context.Context, srcRD string, req *snapshotRestoreRequest, snap *apiv1.Snapshot, eagerPlace bool, overrides *rdShapeOverrides) (materialisedRD, error) { srcRDObj, err := s.Store.ResourceDefinitions().Get(ctx, srcRD) if err != nil { return materialisedRD{}, err //nolint:wrapcheck // surfaced via writeStoreError @@ -574,6 +1347,14 @@ func (s *Server) materializeRestoredRD(ctx context.Context, srcRD string, req *s newRD.Props = store.TravellingProps(srcRDObj.Props) } + overrides.applyTo(&newRD) + + // The stack the restored bytes are brought up under is the source's as + // the data plane resolves it. Left empty, a group named for the target + // whose stack differs would have the control plane judge the definition + // by that group's stack while the satellite brings it up as the source's. + newRD.LayerStack = store.EffectiveLayerStack(newRD.LayerStack) + // Stamp the clone-source so the dispatcher's buildVolumes (called // at every satellite-reconcile of placed Resources) emits // DesiredVolume.SourceSnapshot, which routes the storage provider @@ -581,22 +1362,24 @@ func (s *Server) materializeRestoredRD(ctx context.Context, srcRD string, req *s // `:` is the agreed encoding — satellite splits // on the colon. We persist on the RD (not per-Resource) because // every replica of the new RD clones from the same source. - if newRD.Props == nil { - newRD.Props = map[string]string{} - } - - newRD.Props["BlockstorRestoreFromSnapshot"] = srcRD + ":" + snap.Name + newRD.Props = store.WithRestoreMarker(newRD.Props, snap) - err = s.Store.ResourceDefinitions().Create(ctx, &newRD) + created, adoptedRG, err := s.createOrAdoptRestoredRD(ctx, &newRD, snap, overrides) if err != nil { - return materialisedRD{}, err //nolint:wrapcheck // surfaced via writeStoreError + return materialisedRD{}, err } - made := createdRD(newRD.Name, newRD.ResourceGroupName) + // An adopted leftover is checked against the group it was written with, + // which is the one it carries, not the one this request would have + // stamped: the source may have moved since the first attempt. + made := adoptedRD(newRD.Name, adoptedRG) + if created { + made = createdRD(newRD.Name, newRD.ResourceGroupName) + } - err = hydrateVolumesFromSnapshot(ctx, s, newRD.Name, snap) + err = hydrateVolumesFromSnapshot(ctx, s, newRD.Name, snap, true) if err != nil { - return made, &materialiseAfterCreateError{err: err} + return made, made.failedAfterWrite(err) } // Bug 354: stamp per-node Resource CRDs so satellites have something @@ -607,12 +1390,91 @@ func (s *Server) materializeRestoredRD(ctx context.Context, srcRD string, req *s // empty shell. Mirrors upstream CtrlSnapshotRestoreApiCallHandler. made.Placed, err = s.placeRestoredResources(ctx, srcRD, &newRD, req, snap, eagerPlace) if err != nil { - return made, &materialiseAfterCreateError{err: err} + return made, made.failedAfterWrite(err) } return made, nil } +// createOrAdoptRestoredRD creates the restored definition, and reports whether +// this request created it and, when it adopted one instead, the group that +// leftover carries. +// +// AlreadyExists is tolerated when the definition already there is this +// restore's own — the resume path above, or a second restore of the same +// snapshot racing this one between the state check and here. The marker is +// what tells the two apart from somebody else's definition, and re-reading is +// what makes the decision on fresh state rather than on the read that lost the +// race. +func (s *Server) createOrAdoptRestoredRD(ctx context.Context, newRD *apiv1.ResourceDefinition, snap *apiv1.Snapshot, overrides *rdShapeOverrides) (bool, string, error) { + err := s.Store.ResourceDefinitions().Create(ctx, newRD) + if err == nil { + return true, "", nil + } + + if !errors.Is(err, store.ErrAlreadyExists) { + return false, "", err //nolint:wrapcheck // surfaced via writeStoreError + } + + // The definition that answered AlreadyExists is read from the API server: + // the decision below is whether it is this restore's, and a cache that + // has not seen the marker answers that wrong. + existing, getErr := s.Store.ResourceDefinitions().GetUncached(ctx, newRD.Name) + if getErr != nil { + return false, "", getErr //nolint:wrapcheck // surfaced via writeStoreError + } + + // Re-made on fresh state, and all of it: the marker says the definition + // is this operation's own, the DELETE flag says whether it is still there + // to finish, and the shape says whether it is the same operation. The + // window is narrow — another request completed and the target was deleted + // between the state check above and this Create — but it is the exact + // state the 409 in restoreTargetState exists to prevent, and hydrating + // volumes into a dying definition races the tear-down reaping them. + if !leftoverIsThisRestore(&existing, snap, overrides) { + return false, "", err //nolint:wrapcheck // surfaced via writeStoreError + } + + err = s.claimAdoptedLeftover(ctx, existing.Name, snap) + if err != nil { + return false, "", err + } + + // Hydrated and placed under the name it is stored with: replicas are + // selected by that name exactly, and one stamped under another spelling + // is invisible to the definition's own cascade. + newRD.Name = existing.Name + + return false, existing.ResourceGroupName, nil +} + +// errAdoptedLeftoverRollingBack refuses to adopt a leftover whose creator has +// started rolling it back. +var errAdoptedLeftoverRollingBack = store.ErrAdoptedLeftoverRollingBack + +// claimAdoptedLeftover is the adopting half of a handshake with the attempt +// that created the definition, which "may still be running". +// +// That attempt can fail after this one answered for the definition, and its +// rollback cascades a delete over everything under the name: the caller would +// hold a 201 for a volume that no longer exists. So before writing anything +// into the definition this one marks it as adopted, in one compare-and-swap +// patch that refuses, writing nothing, if a rollback has already marked it +// (store.ClaimAdoptedLeftover). The creator marks its rollback in progress the +// same way and then reads the adoption past the cache (see +// rollBackCompensating). The patches serialise on the definition, so one of +// the two sees the other: a rollback that sees the adoption yields and leaves +// the definition; an adoption that sees the rollback refuses. +func (s *Server) claimAdoptedLeftover(ctx context.Context, rdName string, snap *apiv1.Snapshot) error { + err := store.ClaimAdoptedLeftover(ctx, s.Store, rdName, snap) + if errors.Is(err, store.ErrAdoptedLeftoverRollingBack) { + return errors.Mark(errors.Wrapf(store.ErrAlreadyExists, //nolint:wrapcheck // the mark is the wrap + "'%s' cannot be adopted: %v", rdName, errAdoptedLeftoverRollingBack), errAdoptedLeftoverRollingBack) + } + + return err //nolint:wrapcheck // store names the definition +} + // materialisedRD is what a materialisation wrote, and whether the definition // under that name is this call's to undo. // @@ -626,9 +1488,8 @@ func (s *Server) materializeRestoredRD(ctx context.Context, srcRD string, req *s // unexported origin whose zero value says nothing, and the two constructors are // the only way to state it; a literal that skips them leaves the origin // unstated, and createdHere answers false for it, which is the safe side of the -// line. Every materialisation on this branch creates, so only createdRD is -// called here; the shape exists so a door that starts tolerating a leftover -// cannot hand a compensation someone else's definition by omission. +// line. materializeRestoredRD tolerates this restore's own leftover, so both +// constructors are called there, off what createOrAdoptRestoredRD reports. type materialisedRD struct { // Name is the target definition. Name string @@ -675,6 +1536,18 @@ type materialiseAfterCreateError struct { err error } +// failedAfterWrite is how a materialisation reports a failure after the +// definition stood: as this call's own partial work, which a caller may undo, +// only when this call created it. A leftover it adopted belongs to the attempt +// that left it, and the failure is answered as the store error it is. +func (m materialisedRD) failedAfterWrite(err error) error { + if m.createdHere() { + return &materialiseAfterCreateError{err: err} + } + + return err +} + func (e *materialiseAfterCreateError) Error() string { return e.err.Error() } func (e *materialiseAfterCreateError) Unwrap() error { return e.err } @@ -790,9 +1663,15 @@ func (s *Server) stampRestoredResourcesOnNodes(ctx context.Context, srcRDName, n res.Props = map[string]string{storPoolPropKey: pool} } - err := s.Store.Resources().Create(ctx, &res) + created, err := s.stampRestoredReplica(ctx, &res) if err != nil { - return placed, err //nolint:wrapcheck // wrapped as materialiseAfterCreateError by the caller + return placed, err + } + + if !created { + // Neither created nor promoted by this call, so not in what a + // rollback may take. + continue } placed = append(placed, node) @@ -801,6 +1680,127 @@ func (s *Server) stampRestoredResourcesOnNodes(ctx context.Context, srcRDName, n return placed, nil } +// errReplicaStillDeleting marks a stamp refused because the replica already +// under that (definition, node) is going away. It wraps ErrAlreadyExists, but +// is answered with no FAIL_EXISTS band; see materialiseRefusalKind. +var errReplicaStillDeleting = errors.New("the replica under that name is still being deleted") + +// stampRestoredReplica creates one replica of a restore, and reports whether +// this call created it, or promoted the controller's witness into it. +// +// Same reasoning as hydrateVolumesFromSnapshot: the replica is keyed +// (definition, node) and the caller stamps exactly the nodes the restore +// resolved, so an existing one is the replica a previous attempt already +// placed. Unless it is going away: a Create over a replica still held by its +// satellite finalizer answers AlreadyExists too, and counting that one as +// placed reports a definition with no live replica as done. Nor when it is +// diskless: it holds no copy of the data. A tie-breaker is the witness the +// controller places once two replicas with a disk exist, so on a node the +// restore also places onto it is promoted, the way autoplace promotes one, +// and its volumes are restored from the snapshot like any other replica of a +// marked target; an operator's diskless replica is refused. +// +// The existing replica is read from the API server, not the cache: right after +// a delete began a cache still serves the replica without its DELETE flag, and +// would have it counted as placed. A NotFound on that read is the replica +// having gone between the Create and the read, so the Create is tried again; +// one that keeps colliding with a replica the API server says is not there is +// reported, not taken as placed. +// +// That report cannot come from a store whose read trails its own create. +// Every store this runs against answers both from the same source: the servers +// build theirs with the manager's direct reader (storek8s.NewManager), the CLI +// with an uncached client, and the in-memory store has no cache. A store built +// with a cached client and no direct reader would read past nothing here, and +// is not a configuration any binary ships. +func (s *Server) stampRestoredReplica(ctx context.Context, res *apiv1.Resource) (bool, error) { + const attempts = 3 + + for attempt := 1; ; attempt++ { + err := s.Store.Resources().Create(ctx, res) + if err == nil { + return true, nil + } + + if !errors.Is(err, store.ErrAlreadyExists) { + return false, err //nolint:wrapcheck // wrapped as materialiseAfterCreateError by the caller + } + + existing, getErr := getResourceUncached(ctx, s.Store, res.Name, res.NodeName) + if errors.Is(getErr, store.ErrNotFound) { + if attempt < attempts { + continue + } + + return false, errors.Wrapf(store.ErrAlreadyExists, + "the replica of %q on node %q keeps colliding with one the API server does not have", + res.Name, res.NodeName) + } + + if getErr != nil { + return false, getErr + } + + if replicaAcceptedForDeletion(&existing) { + return false, errors.Mark(errors.Wrapf(store.ErrAlreadyExists, //nolint:wrapcheck // the mark is the wrap + "the replica of %q on node %q is still being deleted", res.Name, res.NodeName), + errReplicaStillDeleting) + } + + // A promoted witness is this request's replica now, and counts as + // placed: a rollback deletes it by name like any replica it made. + promotedHere := false + + if slices.Contains(existing.Flags, apiv1.ResourceFlagTieBreaker) { + promoted, promoteErr := s.promoteRestoreWitness(ctx, res) + if promoteErr != nil { + return false, promoteErr + } + + existing, promotedHere = *promoted, true + } + + if !store.HoldsData(&existing) { + return false, errors.Wrapf(store.ErrAlreadyExists, + "the replica of %q on node %q is diskless and holds no copy of the data; "+ + "delete it, then retry", res.Name, res.NodeName) + } + + return promotedHere, nil + } +} + +// promoteRestoreWitness promotes the controller's witness on a node the restore +// places onto. The pool is settled before anything is written: a promotion +// that found none would strip the witness's TIE_BREAKER flag and leave it +// looking like an operator's diskless replica, refused on every retry. With no +// pool the witness is left untouched and the stamp is refused. +func (s *Server) promoteRestoreWitness(ctx context.Context, res *apiv1.Resource) (*apiv1.Resource, error) { + target := *res + if target.Props["StorPoolName"] == "" { + pool, err := s.resolveTakeoverStorPool(ctx, res.Name, res.NodeName) + if err != nil { + return nil, err + } + + if pool == "" { + return nil, errors.Wrapf(store.ErrAlreadyExists, + "the witness of %q on node %q has no storage pool to take a disk in; give it one with "+ + "`linstor resource create --storage-pool %s %s`, then retry", res.Name, res.NodeName, + res.NodeName, res.Name) + } + + target.Props = maps.Clone(res.Props) + if target.Props == nil { + target.Props = map[string]string{} + } + + target.Props["StorPoolName"] = pool + } + + return s.promoteDisklessReplica(ctx, &target) +} + // canonicalRestoreNodeList collapses the request's two node-list // aliases (`nodes` and `node_names`) into a single ordered list. // Both wire shapes appear in the wild: upstream LINSTOR CLI/golinstor @@ -859,7 +1859,16 @@ func storPoolsByNodeFromSourceRD(ctx context.Context, st store.Store, srcRDName // autoplace creates empty Resources that never reach UpToDate. // linstor-csi's CreateVolume-from-source path relies on this // hydration to surface the cloned PVC's block device. -func hydrateVolumesFromSnapshot(ctx context.Context, s *Server, rdName string, snap *apiv1.Snapshot) error { +// +// ownTarget says the definition carries this restore's or clone's own marker, +// which materializeRestoredRD has established before it calls this. Only then +// is a volume LARGER than the snapshot recorded this operation's own: it is +// the one an earlier attempt hydrated, expanded since like any volume. +// leftoverAgainstSnapshot classifies that shape as the clone's own, and the +// resume it admits ends here, so refusing it answered a bare 500 on every +// retry. The volume-definition restore hydrates into a definition that is +// not its own and accepts no existing volume at all. +func hydrateVolumesFromSnapshot(ctx context.Context, s *Server, rdName string, snap *apiv1.Snapshot, ownTarget bool) error { for i := range snap.VolumeDefinitions { svd := &snap.VolumeDefinitions[i] vd := apiv1.VolumeDefinition{ @@ -868,7 +1877,32 @@ func hydrateVolumesFromSnapshot(ctx context.Context, s *Server, rdName string, s } err := s.Store.VolumeDefinitions().Create(ctx, rdName, &vd) - if err != nil { + if err == nil { + continue + } + + // The volume-definition restore writes into a definition that is not + // its own, and every collision there is a refusal, as on main: it is + // the second half of that restore's collision guard, and a volume + // that appeared between the guard's list and this create was written + // by somebody else. A concurrent CLI restore of the same snapshot can + // fail on a later volume and unwind this one, so a 200 over it would + // report a layout that is about to go. + if !errors.Is(err, store.ErrAlreadyExists) || !ownTarget { + return err //nolint:wrapcheck // wrapped as materialiseAfterCreateError by the caller + } + + // On this operation's own target, under its marker and the adoption + // handshake, AlreadyExists is the volume an earlier attempt hydrated, + // and tolerating it is what lets a retry finish an incomplete restore. + // The size still tells it apart: one at the snapshot's size, or grown + // since, is the restore's own; a smaller one is somebody else's. + existing, getErr := s.Store.VolumeDefinitions().Get(ctx, rdName, svd.VolumeNumber) + if getErr != nil { + return err //nolint:wrapcheck // the collision is the answer, not the read + } + + if existing.SizeKib < svd.SizeKib { return err //nolint:wrapcheck // wrapped as materialiseAfterCreateError by the caller } } diff --git a/pkg/rest/snapshot_restore_csi_flow_test.go b/pkg/rest/snapshot_restore_csi_flow_test.go new file mode 100644 index 00000000..6c4e61a8 --- /dev/null +++ b/pkg/rest/snapshot_restore_csi_flow_test.go @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "encoding/json" + "net/http" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// csiPrepareRestoreTarget runs the first two calls linstor-csi v1.11.2's +// VolFromSnap makes before it asks for the resources: it creates the +// definition itself (reconcileResourceDefinition), then restores the volume +// definitions onto it when it has none (reconcileSnapshotVolumeDefinitions). +func csiPrepareRestoreTarget(t *testing.T, base, src, snap, target string) { + t.Helper() + + csiPrepareRestoreTargetWithLayers(t, base, src, snap, target, nil) +} + +// csiPrepareRestoreTargetWithLayers is csiPrepareRestoreTarget for a +// StorageClass that names a layer list, which linstor-csi sends on the +// definition it creates. +func csiPrepareRestoreTargetWithLayers(t *testing.T, base, src, snap, target string, layers []string) { + t.Helper() + + create := map[string]any{"resource_definition": map[string]any{"name": target}} + if len(layers) > 0 { + create["layer_list"] = layers + } + + body, err := json.Marshal(create) + if err != nil { + t.Fatalf("marshal rd create: %v", err) + } + + resp := httpPost(t, base+"/v1/resource-definitions", body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("rd create (reconcileResourceDefinition) = %d, want 201", resp.StatusCode) + } + + raw, err := json.Marshal(map[string]any{"to_resource": target}) + if err != nil { + t.Fatalf("marshal vd restore: %v", err) + } + + resp = httpPost(t, base+"/v1/resource-definitions/"+src+"/snapshot-restore-volume-definition/"+snap, raw) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("vd restore (reconcileSnapshotVolumeDefinitions) = %d, want 200", resp.StatusCode) + } +} + +// linstor-csi creates the target definition itself and restores its volume +// definitions before it asks for the resources, so the definition never +// carried this restore's marker and the resource restore refused it as +// somebody else's: every volume restored from a snapshot through CSI failed on +// its first call. +func TestSnapshotRestoreFollowsLinstorCSIsOwnSequence(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-csi") + seedRestoreSnapshot(t, st, "src-csi", "snap-csi", []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + csiPrepareRestoreTarget(t, base, "src-csi", "snap-csi", "pvc-csi") + + // reconcileSnapshotResources: one node that holds the snapshot. + if code := restoreOnce(t, base, "src-csi", "snap-csi", map[string]any{ + "to_resource": "pvc-csi", "nodes": []string{"node-a"}, + }); code != http.StatusCreated { + t.Fatalf("resource restore over the definition CSI prepared = %d, want 201", code) + } + + rd, err := st.ResourceDefinitions().Get(ctx, "pvc-csi") + if err != nil { + t.Fatalf("read the restored definition: %v", err) + } + + if got, want := rd.Props[restoreFromSnapshotKey], restoreMarker("src-csi", "snap-csi"); got != want { + t.Errorf("marker = %q, want %q: the satellite restores a replica from the snapshot only through it", got, want) + } + + replicas, err := st.Resources().ListByDefinition(ctx, "pvc-csi") + if err != nil || len(replicas) != 1 || replicas[0].NodeName != "node-a" { + t.Errorf("replicas after the restore = %v (%v), want one on node-a", replicas, err) + } +} + +// CSI retries the whole chain when a step fails. Its own checks skip the +// definition and volume steps once they are done; the resource restore it +// re-issues lands on a target it already marked, and finishes it. +func TestSnapshotRestoreFinishesLinstorCSIsRetriedChain(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-csr") + seedRestoreSnapshot(t, st, "src-csr", "snap-csr", []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + csiPrepareRestoreTarget(t, base, "src-csr", "snap-csr", "pvc-csr") + + body := map[string]any{"to_resource": "pvc-csr", "nodes": []string{"node-a"}} + + if code := restoreOnce(t, base, "src-csr", "snap-csr", body); code != http.StatusCreated { + t.Fatalf("first resource restore = %d, want 201", code) + } + + // The first attempt's replica did not survive; CSI's next pass finds + // the definition and its volumes and asks for the resources again. + if err := st.Resources().Delete(ctx, "pvc-csr", "node-a"); err != nil { + t.Fatalf("drop the replica: %v", err) + } + + if code := restoreOnce(t, base, "src-csr", "snap-csr", body); code != http.StatusCreated { + t.Fatalf("retried resource restore = %d, want 201", code) + } + + if replicas, _ := st.Resources().ListByDefinition(ctx, "pvc-csr"); len(replicas) != 1 { + t.Errorf("replicas after the retried chain = %d, want 1", len(replicas)) + } +} + +// What the caller prepared is taken only in the state CSI leaves. A definition +// with a live replica may hold data, and one whose volumes are not exactly the +// snapshot's was not prepared from it. +func TestSnapshotRestoreRefusesATargetNotPreparedFromTheSnapshot(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + seed func(t *testing.T, st store.Store, target string) + }{ + {name: "live-replica", seed: func(t *testing.T, st store.Store, target string) { + t.Helper() + + if err := st.VolumeDefinitions().Create(t.Context(), target, + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the volume: %v", err) + } + + if err := st.Resources().Create(t.Context(), &apiv1.Resource{Name: target, NodeName: "node-a"}); err != nil { + t.Fatalf("seed the live replica: %v", err) + } + }}, + {name: "other-size", seed: func(t *testing.T, st store.Store, target string) { + t.Helper() + + if err := st.VolumeDefinitions().Create(t.Context(), target, + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 128 * 1024}); err != nil { + t.Fatalf("seed the volume: %v", err) + } + }}, + {name: "no-volumes", seed: func(*testing.T, store.Store, string) {}}, + {name: "being-deleted", seed: func(t *testing.T, st store.Store, target string) { + t.Helper() + + seedSnapshotShapedVolume(t, st, target) + + rd, err := st.ResourceDefinitions().Get(t.Context(), target) + if err != nil { + t.Fatalf("read the target: %v", err) + } + + rd.Flags = append(rd.Flags, apiv1.ResourceFlagDelete) + if err := st.ResourceDefinitions().Update(t.Context(), &rd); err != nil { + t.Fatalf("flag the target for deletion: %v", err) + } + }}, + {name: "extra-volume", seed: func(t *testing.T, st store.Store, target string) { + t.Helper() + + seedSnapshotShapedVolume(t, st, target) + + if err := st.VolumeDefinitions().Create(t.Context(), target, + &apiv1.VolumeDefinition{VolumeNumber: 1, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the volume the snapshot never held: %v", err) + } + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + target := "pvc-np-" + tc.name + seedDeployedCloneSource(t, st, "src-np") + seedRestoreSnapshot(t, st, "src-np", "snap-np", []string{"node-a"}) + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: target}); err != nil { + t.Fatalf("seed the target: %v", err) + } + + tc.seed(t, st, target) + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := restoreOnce(t, base, "src-np", "snap-np", map[string]any{ + "to_resource": target, "nodes": []string{"node-a"}, + }); code != http.StatusConflict { + t.Errorf("restore into a target not prepared from the snapshot = %d, want 409", code) + } + + rd, err := st.ResourceDefinitions().Get(ctx, target) + if err != nil { + t.Fatalf("read the target: %v", err) + } + + if marker := rd.Props[restoreFromSnapshotKey]; marker != "" { + t.Errorf("the refused target was marked %q", marker) + } + }) + } +} + +// seedSnapshotShapedVolume gives a prepared target the one volume the +// snapshot holds, at its size, so a case differs from a usable target only in +// what it adds. +func seedSnapshotShapedVolume(t *testing.T, st store.Store, target string) { + t.Helper() + + if err := st.VolumeDefinitions().Create(t.Context(), target, + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the volume: %v", err) + } +} diff --git a/pkg/rest/snapshot_restore_idempotency_test.go b/pkg/rest/snapshot_restore_idempotency_test.go new file mode 100644 index 00000000..412f7d31 --- /dev/null +++ b/pkg/rest/snapshot_restore_idempotency_test.go @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// decodeRCs reads the []ApiCallRc envelope the restore endpoint answers with. +func decodeRCs(t *testing.T, resp *http.Response) []apiv1.APICallRc { + t.Helper() + + var rcs []apiv1.APICallRc + if err := json.NewDecoder(resp.Body).Decode(&rcs); err != nil { + t.Fatalf("decode the envelope: %v", err) + } + + if len(rcs) == 0 { + t.Fatal("the endpoint returned an empty envelope") + } + + return rcs +} + +// seedRestoreSource is a definition with one snapshot of it, the minimum a +// snapshot-restore needs. +// drbdRestoreMarkerForTest is the marker key materializeRestoredRD stamps. +const drbdRestoreMarkerForTest = "BlockstorRestoreFromSnapshot" + +func seedRestoreSource(ctx context.Context, t *testing.T, st store.Store) { + t.Helper() + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "pvc-src"}); err != nil { + t.Fatalf("seed RD: %v", err) + } + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "snap-1", + ResourceName: "pvc-src", + Nodes: []string{"n1", "n2"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{ + {VolumeNumber: 0, SizeKib: 1024 * 1024}, + }, + }); err != nil { + t.Fatalf("seed snap: %v", err) + } +} + +// CSI requires CreateVolume to be idempotent: a repeat with the same name and +// parameters must succeed and return the volume that already exists. +// external-provisioner has no other way to make progress after a partial +// failure, so a second call that fails makes the first partial failure +// terminal for that volume name — the PVC stays Pending forever and the +// leftover definition has to be deleted by hand. +func TestSnapshotRestoreIsIdempotent(t *testing.T) { + st := store.NewInMemory() + ctx := t.Context() + seedRestoreSource(ctx, t, st) + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(map[string]string{ + "to_resource": "pvc-dst", + "from_snapshot": "snap-1", + }) + + first := httpPost(t, base+"/v1/resource-definitions/pvc-src/snapshot-restore-resource", body) + _ = first.Body.Close() + + if first.StatusCode != http.StatusCreated { + t.Fatalf("first restore: got %d, want 201", first.StatusCode) + } + + // The retry external-provisioner issues after a partial failure. + second := httpPost(t, base+"/v1/resource-definitions/pvc-src/snapshot-restore-resource", body) + defer func() { _ = second.Body.Close() }() + + if second.StatusCode != http.StatusCreated { + t.Fatalf("retry: got %d, want 201 — a repeat of the same restore must succeed, "+ + "or the first partial failure is terminal for this volume name", second.StatusCode) + } + + var rcs []apiv1.APICallRc + if err := json.NewDecoder(second.Body).Decode(&rcs); err != nil { + t.Fatalf("decode retry envelope: %v", err) + } + + if len(rcs) == 0 { + t.Fatal("the retry returned an empty envelope") + } +} + +// Idempotency stops at the restore's own leftover. A definition under that +// name which is NOT the one this restore would have produced is a genuine +// collision, and answering 201 would report a restore that never happened +// over somebody else's data. +func TestSnapshotRestoreStillRefusesAForeignName(t *testing.T) { + st := store.NewInMemory() + ctx := t.Context() + seedRestoreSource(ctx, t, st) + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-dst", + Props: map[string]string{"someone": "else"}, + }); err != nil { + t.Fatalf("seed the occupant: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(map[string]string{ + "to_resource": "pvc-dst", + "from_snapshot": "snap-1", + }) + + resp := httpPost(t, base+"/v1/resource-definitions/pvc-src/snapshot-restore-resource", body) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status: got %d, want 409 — that name holds a definition this restore "+ + "did not create", resp.StatusCode) + } + + // The refusal is this endpoint's own, not whatever the store write would + // have surfaced on its way out: upstream callers switch on the mask, and + // python-linstor prints the correction. + rc := decodeRCs(t, resp)[0] + + if rc.RetCode&apiCallRcFailExistsRscDfn == 0 { + t.Errorf("ret_code = %d, want the FAIL_EXISTS_RSC_DFN mask set", rc.RetCode) + } + + if !strings.Contains(rc.Message, "is not a restore of") { + t.Errorf("message = %q, want it to say the name holds something else", rc.Message) + } + + got, err := st.ResourceDefinitions().Get(ctx, "pvc-dst") + if err != nil { + t.Fatalf("get the occupant: %v", err) + } + + if got.Props["someone"] != "else" { + t.Error("the refused restore overwrote the definition that was already there") + } +} + +// The marker is stamped with the definition, BEFORE its volumes are hydrated +// and its replicas placed. So a definition carrying it says a restore +// started, not that one finished, and a retry that reads the marker as +// completion turns the terminal failure this endpoint used to have into a +// silent incomplete one — CSI sees the volume as ready and nothing ever +// finishes it. +// +// The leftover here is exactly that shape: the definition and the marker, no +// volumes. The retry has to complete it. +func TestSnapshotRestoreResumesAnIncompleteLeftover(t *testing.T) { + st := store.NewInMemory() + ctx := t.Context() + seedRestoreSource(ctx, t, st) + + // What a first attempt leaves behind when it fails after creating the + // definition and before hydrating the volumes. + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-dst", + Props: map[string]string{drbdRestoreMarkerForTest: "pvc-src:snap-1"}, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + vds, err := st.VolumeDefinitions().List(ctx, "pvc-dst") + if err != nil { + t.Fatalf("list the leftover's volumes: %v", err) + } + + if len(vds) != 0 { + t.Fatalf("the leftover already has %d volume(s); the test is not modelling "+ + "an incomplete restore", len(vds)) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(map[string]string{ + "to_resource": "pvc-dst", + "from_snapshot": "snap-1", + }) + + resp := httpPost(t, base+"/v1/resource-definitions/pvc-src/snapshot-restore-resource", body) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("retry over an incomplete leftover: got %d, want 201", resp.StatusCode) + } + + // The point of the retry: the volumes the first attempt never wrote. + vds, err = st.VolumeDefinitions().List(ctx, "pvc-dst") + if err != nil { + t.Fatalf("list volumes after the retry: %v", err) + } + + if len(vds) != 1 { + t.Fatalf("after the retry the target has %d volume(s), want 1 — the retry "+ + "reported success over a definition it never finished", len(vds)) + } + + // And it says so, rather than reading like a first run. + if msg := decodeRCs(t, resp)[0].Message; !strings.Contains(msg, "on retry") { + t.Errorf("message = %q, want it to name the retry", msg) + } +} + +// Resuming stops at a leftover being torn down. The DELETE flag is how a +// definition with a DeletionTimestamp reaches this shim +// (pkg/store/k8s/resource_definitions.go), and finishing a restore into one +// races the tear-down reaping the volumes the retry is writing. +func TestSnapshotRestoreRefusesALeftoverBeingDeleted(t *testing.T) { + st := store.NewInMemory() + ctx := t.Context() + seedRestoreSource(ctx, t, st) + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-dst", + Flags: []string{rdFlagDelete}, + Props: map[string]string{drbdRestoreMarkerForTest: "pvc-src:snap-1"}, + }); err != nil { + t.Fatalf("seed the dying leftover: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(map[string]string{ + "to_resource": "pvc-dst", + "from_snapshot": "snap-1", + }) + + resp := httpPost(t, base+"/v1/resource-definitions/pvc-src/snapshot-restore-resource", body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status: got %d, want 409 — that leftover is being deleted", resp.StatusCode) + } + + vds, err := st.VolumeDefinitions().List(ctx, "pvc-dst") + if err != nil { + t.Fatalf("list the leftover's volumes: %v", err) + } + + if len(vds) != 0 { + t.Errorf("the refused restore hydrated %d volume(s) into a definition being deleted", len(vds)) + } +} + +// caseFoldingStore models the one thing about the real store this shim runs +// on that the in-memory one does not: LINSTOR names fold case, so +// pkg/store/k8s/crdname.go resolves `SNAP-1` and `snap-1` to the same object +// and hands back whichever spelling was stored. +// +// Both kinds fold, because both halves of the marker are names: a definition +// looked up under one spelling comes back carrying the one it was stored with, +// and so does a snapshot. +type caseFoldingStore struct{ store.Store } + +func (s caseFoldingStore) Snapshots() store.SnapshotStore { + return caseFoldingSnapshots{s.Store.Snapshots()} +} + +func (s caseFoldingStore) ResourceDefinitions() store.ResourceDefinitionStore { + return caseFoldingRDs{s.Store.ResourceDefinitions()} +} + +func (s caseFoldingStore) VolumeDefinitions() store.VolumeDefinitionStore { + return caseFoldingVDs{s.Store.VolumeDefinitions()} +} + +type caseFoldingSnapshots struct{ store.SnapshotStore } + +func (s caseFoldingSnapshots) Get(ctx context.Context, rdName, snapName string) (apiv1.Snapshot, error) { + return s.SnapshotStore.Get(ctx, strings.ToLower(rdName), strings.ToLower(snapName)) //nolint:wrapcheck // pass-through decorator +} + +func (s caseFoldingSnapshots) Create(ctx context.Context, snap *apiv1.Snapshot) error { + folded := *snap + folded.Name = strings.ToLower(snap.Name) + folded.ResourceName = strings.ToLower(snap.ResourceName) + + return s.SnapshotStore.Create(ctx, &folded) //nolint:wrapcheck // pass-through decorator +} + +type caseFoldingRDs struct{ store.ResourceDefinitionStore } + +func (s caseFoldingRDs) Get(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + return s.ResourceDefinitionStore.Get(ctx, strings.ToLower(name)) //nolint:wrapcheck // pass-through decorator +} + +// GetUncached folds the same way: the real store's live read goes through the +// same name translation as its cached one. +func (s caseFoldingRDs) GetUncached(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + return s.ResourceDefinitionStore.GetUncached(ctx, strings.ToLower(name)) //nolint:wrapcheck // pass-through decorator +} + +// Create folds too, because the real store folds the metadata.name it writes, +// so a create under a differently-cased name collides there. A shim that +// folded only lookups let a retry create a SECOND definition beside the +// leftover and still satisfy every assertion, which left the tolerance branch +// these tests exist to pin unreached. +func (s caseFoldingRDs) Create(ctx context.Context, rd *apiv1.ResourceDefinition) error { + folded := *rd + folded.Name = strings.ToLower(rd.Name) + + return s.ResourceDefinitionStore.Create(ctx, &folded) //nolint:wrapcheck // pass-through decorator +} + +type caseFoldingVDs struct{ store.VolumeDefinitionStore } + +func (s caseFoldingVDs) List(ctx context.Context, rdName string) ([]apiv1.VolumeDefinition, error) { + return s.VolumeDefinitionStore.List(ctx, strings.ToLower(rdName)) //nolint:wrapcheck // pass-through decorator +} + +func (s caseFoldingVDs) Create(ctx context.Context, rdName string, vd *apiv1.VolumeDefinition) error { + return s.VolumeDefinitionStore.Create(ctx, strings.ToLower(rdName), vd) //nolint:wrapcheck // pass-through decorator +} + +// The retry has to be recognised whichever way the caller spelled the names. +// The marker is written off the stored snapshot, so comparing it against the +// request's own spelling made `--from-snapshot SNAP-1` read as somebody else's +// definition and get refused — the terminal-on-first-failure behaviour the +// resume path exists to end, back again for any caller that does not echo the +// case byte for byte. +func TestSnapshotRestoreResumesWhateverCaseTheRetryUses(t *testing.T) { + st := caseFoldingStore{store.NewInMemory()} + ctx := t.Context() + seedRestoreSource(ctx, t, st) + + // What the first attempt left: the marker carries the STORED spelling. + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-dst", + Props: map[string]string{drbdRestoreMarkerForTest: "pvc-src:snap-1"}, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(map[string]string{ + "to_resource": "pvc-dst", + "from_snapshot": "SNAP-1", + }) + + resp := httpPost(t, base+"/v1/resource-definitions/pvc-src/snapshot-restore-resource", body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("retry spelled `SNAP-1` over a marker written `snap-1`: got %d, want 201", + resp.StatusCode) + } + + vds, err := st.VolumeDefinitions().List(ctx, "pvc-dst") + if err != nil { + t.Fatalf("list volumes after the retry: %v", err) + } + + if len(vds) != 1 { + t.Errorf("after the retry the target has %d volume(s), want 1", len(vds)) + } + + // Exactly one, which is the assertion that makes the fold load-bearing: a + // shim folding only lookups let the retry create a second definition + // beside the leftover and pass everything else. + all, err := st.ResourceDefinitions().List(ctx) + if err != nil { + t.Fatalf("list definitions: %v", err) + } + + targets := 0 + + for i := range all { + if strings.EqualFold(all[i].Name, "pvc-dst") { + targets++ + } + } + + if targets != 1 { + t.Errorf("%d definitions under the target name; the retry created a second one "+ + "beside the leftover instead of resuming it", targets) + } +} + +// The marker's SOURCE half is written from the stored spelling, and nothing +// held it: every shipped restore test spells the source the way it is stored, +// and the case-fold resume test varies only the snapshot half. So reverting +// that half to the request's spelling left the package green, and the fix +// could regress in silence. +// +// The satellite splits this value to find the source, and a retry compares +// against it, so the spelling written is not cosmetic. +func TestSnapshotRestoreWritesTheMarkerWithTheStoredSourceSpelling(t *testing.T) { + st := caseFoldingStore{store.NewInMemory()} + ctx := t.Context() + seedRestoreSource(ctx, t, st) + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(map[string]string{ + "to_resource": "pvc-dst", + "from_snapshot": "snap-1", + }) + + // The source spelled the way an operator might type it, not the way it is + // stored. + resp := httpPost(t, base+"/v1/resource-definitions/PVC-SRC/snapshot-restore-resource", body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("restore spelled PVC-SRC: got %d, want 201", resp.StatusCode) + } + + got, err := st.ResourceDefinitions().Get(ctx, "pvc-dst") + if err != nil { + t.Fatalf("get the restored definition: %v", err) + } + + if want := restoreMarker("pvc-src", "snap-1"); got.Props[restoreFromSnapshotKey] != want { + t.Errorf("marker = %q, want %q — both halves come off the stored objects, "+ + "not off the request", got.Props[restoreFromSnapshotKey], want) + } +} diff --git a/pkg/rest/snapshot_restore_prepared_cache_test.go b/pkg/rest/snapshot_restore_prepared_cache_test.go new file mode 100644 index 00000000..77dfa202 --- /dev/null +++ b/pkg/rest/snapshot_restore_prepared_cache_test.go @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "context" + "maps" + "net/http" + "strings" + "sync/atomic" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// markerBehindCache serves cached reads of a definition without the restore +// marker, the way a controller-runtime cache does until the watch delivers +// the request's own patch. Uncached reads and writes reach the backend. +type markerBehindCache struct{ store.Store } + +type markerBehindCacheRDs struct{ store.ResourceDefinitionStore } + +func (m markerBehindCache) ResourceDefinitions() store.ResourceDefinitionStore { + return markerBehindCacheRDs{m.Store.ResourceDefinitions()} +} + +func (r markerBehindCacheRDs) Get(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + rd, err := r.ResourceDefinitionStore.Get(ctx, name) + rd.Props = maps.Clone(rd.Props) + delete(rd.Props, store.RestoreFromSnapshotProp) + + return rd, err //nolint:wrapcheck // pass-through test double +} + +// The prepared path stamps the marker through the API server and the same +// request then judged the target through the cache, which had not seen the +// stamp: linstor-csi's first resource restore was refused as somebody else's. +func TestSnapshotRestoreIntoAPreparedTargetDoesNotReadItsOwnMarkerFromTheCache(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, backend, "src-lag") + seedRestoreSnapshot(t, backend, "src-lag", "snap-lag", []string{"node-a"}) + + base, stop := startServerWithStore(t, markerBehindCache{backend}) + defer stop() + + csiPrepareRestoreTarget(t, base, "src-lag", "snap-lag", "pvc-lag") + + if code := restoreOnce(t, base, "src-lag", "snap-lag", map[string]any{ + "to_resource": "pvc-lag", "nodes": []string{"node-a"}, + }); code != http.StatusCreated { + t.Fatalf("first CSI resource restore with the marker not yet in the cache = %d, want 201", code) + } + + replicas, err := backend.Resources().ListByDefinition(ctx, "pvc-lag") + if err != nil || len(replicas) != 1 || replicas[0].NodeName != "node-a" { + t.Errorf("replicas after the restore = %v (%v), want one on node-a", replicas, err) + } +} + +// replicaAppearsOnStamp places a live replica on the requested node just after +// the restore marker is patched onto the target: what a concurrent retry of the +// same restore does once it finds the marker this request wrote. +type replicaAppearsOnStamp struct { + store.Store + + target, node string +} + +type replicaAppearsOnStampRDs struct { + store.ResourceDefinitionStore + + outer replicaAppearsOnStamp +} + +func (r replicaAppearsOnStamp) ResourceDefinitions() store.ResourceDefinitionStore { + return replicaAppearsOnStampRDs{ResourceDefinitionStore: r.Store.ResourceDefinitions(), outer: r} +} + +func (r replicaAppearsOnStampRDs) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + err := r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) + if err == nil && strings.EqualFold(name, r.outer.target) { + _ = r.outer.Store.Resources().Create(ctx, &apiv1.Resource{Name: name, NodeName: r.outer.node}) + } + + return err //nolint:wrapcheck // pass-through test double +} + +// A replica that appears once the marker is on is what a concurrent retry of +// the same restore places. Refusing over it and taking the marker back off +// left a definition another request had already answered for with no marker. +func TestSnapshotRestoreKeepsItsMarkerOverAReplicaARetryPlaced(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, backend, "src-gain") + seedRestoreSnapshot(t, backend, "src-gain", "snap-gain", []string{"node-a"}) + + plain, stopPlain := startServerWithStore(t, backend) + csiPrepareRestoreTarget(t, plain, "src-gain", "snap-gain", "pvc-gain") + stopPlain() + + base, stop := startServerWithStore(t, replicaAppearsOnStamp{Store: backend, target: "pvc-gain", node: "node-a"}) + defer stop() + + if code := restoreOnce(t, base, "src-gain", "snap-gain", map[string]any{ + "to_resource": "pvc-gain", "nodes": []string{"node-a"}, + }); code != http.StatusCreated { + t.Errorf("restore into a target a concurrent retry placed a replica on = %d, want 201", code) + } + + rd, err := backend.ResourceDefinitions().Get(ctx, "pvc-gain") + if err != nil { + t.Fatalf("read the target: %v", err) + } + + if rd.Props[store.RestoreFromSnapshotProp] != restoreMarker("src-gain", "snap-gain") { + t.Errorf("the restore marker = %q, want it kept", rd.Props[store.RestoreFromSnapshotProp]) + } + + if replicas, _ := backend.Resources().ListByDefinition(ctx, "pvc-gain"); len(replicas) != 1 { + t.Errorf("replicas = %v, want the one on node-a", replicas) + } +} + +// holdAfterMarking holds the first request that stamps the restore marker on +// the target right after the stamp, until released: long enough for a +// concurrent retry of the same restore to run start to finish. +type holdAfterMarking struct { + store.Store + + target string + first *atomic.Bool + stamped chan struct{} + release chan struct{} +} + +type holdAfterMarkingRDs struct { + store.ResourceDefinitionStore + + outer holdAfterMarking +} + +func (h holdAfterMarking) ResourceDefinitions() store.ResourceDefinitionStore { + return holdAfterMarkingRDs{ResourceDefinitionStore: h.Store.ResourceDefinitions(), outer: h} +} + +func (h holdAfterMarkingRDs) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + err := h.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) + if err == nil && strings.EqualFold(name, h.outer.target) { + // Only the first stamp is held. A sync.Once would hold every later + // patch too, the concurrent retry's own among them, until release. + if h.outer.first.CompareAndSwap(false, true) { + close(h.outer.stamped) + <-h.outer.release + } + } + + return err //nolint:wrapcheck // pass-through test double +} + +// linstor-csi re-issues CreateVolume on a timeout. The retry found the marker +// the first request had just written, resumed, placed the replica and answered +// 201; the first request then took that replica for somebody else's, removed +// the marker and answered 409, leaving the definition CSI was told is restored +// with no marker. +func TestSnapshotRestoreSurvivesAConcurrentRetryOfTheSameRestore(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, backend, "src-twin") + seedRestoreSnapshot(t, backend, "src-twin", "snap-twin", []string{"node-a"}) + + plain, stopPlain := startServerWithStore(t, backend) + csiPrepareRestoreTarget(t, plain, "src-twin", "snap-twin", "pvc-twin") + stopPlain() + + held := holdAfterMarking{ + Store: backend, target: "pvc-twin", first: &atomic.Bool{}, + stamped: make(chan struct{}), release: make(chan struct{}), + } + + base, stop := startServerWithStore(t, held) + defer stop() + + body := map[string]any{"to_resource": "pvc-twin", "nodes": []string{"node-a"}} + first := make(chan int, 1) + + go func() { first <- restoreOnce(t, base, "src-twin", "snap-twin", body) }() + + <-held.stamped + + if code := restoreOnce(t, base, "src-twin", "snap-twin", body); code != http.StatusCreated { + t.Errorf("the concurrent retry = %d, want 201", code) + } + + close(held.release) + + if code := <-first; code != http.StatusCreated { + t.Errorf("the request that stamped the marker = %d, want 201", code) + } + + rd, err := backend.ResourceDefinitions().Get(ctx, "pvc-twin") + if err != nil { + t.Fatalf("read the target: %v", err) + } + + if rd.Props[store.RestoreFromSnapshotProp] != restoreMarker("src-twin", "snap-twin") { + t.Errorf("the restore marker = %q, want it kept", rd.Props[store.RestoreFromSnapshotProp]) + } + + if replicas, _ := backend.Resources().ListByDefinition(ctx, "pvc-twin"); len(replicas) != 1 { + t.Errorf("replicas = %v, want exactly one on node-a", replicas) + } +} diff --git a/pkg/rest/snapshot_restore_prepared_target_test.go b/pkg/rest/snapshot_restore_prepared_target_test.go new file mode 100644 index 00000000..08fe16c4 --- /dev/null +++ b/pkg/rest/snapshot_restore_prepared_target_test.go @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "net/http" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +func assertTargetUntouched(t *testing.T, st store.Store, target string) { + t.Helper() + + rd, err := st.ResourceDefinitions().Get(t.Context(), target) + if err != nil { + t.Fatalf("read the target: %v", err) + } + + if marker := rd.Props[restoreFromSnapshotKey]; marker != "" { + t.Errorf("the refused target was marked %q", marker) + } + + replicas, err := st.Resources().ListByDefinition(t.Context(), target) + if err != nil { + t.Fatalf("list the target's replicas: %v", err) + } + + for i := range replicas { + if !replicaAcceptedForDeletion(&replicas[i]) { + t.Errorf("a replica was placed on the refused target on %s", replicas[i].NodeName) + } + } +} + +// A restore puts the source's bytes under the target's stack. A layer the +// source did not have writes its own metadata across them, and one it had is +// missing on read; through linstor-csi this is a VolumeSnapshot restored into +// a PVC of another StorageClass. +func TestSnapshotRestoreRefusesAPreparedTargetOfAnotherLayerStack(t *testing.T) { + t.Parallel() + + luks := []string{"DRBD", "LUKS", "STORAGE"} + + for _, tc := range []struct { + name string + source, class []string + }{ + {name: "plaintext-source-into-luks", class: luks}, + {name: "luks-source-into-plaintext", source: luks}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedCloneSourceWithStack(t, st, "src-layers", tc.source) + seedRestoreSnapshot(t, st, "src-layers", "snap-layers", []string{"node-a"}) + + // A LUKS definition is created only on a cluster with a passphrase. + base, stop := startServerWithPassphrase(t, st) + defer stop() + + csiPrepareRestoreTargetWithLayers(t, base, "src-layers", "snap-layers", "pvc-layers", tc.class) + + if code := restoreOnce(t, base, "src-layers", "snap-layers", map[string]any{ + "to_resource": "pvc-layers", "nodes": []string{"node-a"}, + }); code != http.StatusConflict { + t.Errorf("restore into a target of another layer stack = %d, want 409", code) + } + + assertTargetUntouched(t, st, "pvc-layers") + }) + } +} + +// Placing over replicas still being deleted races the tear-down, and the +// refusal comes before the marker goes on a definition that is the caller's. +func TestSnapshotRestoreRefusesAPreparedTargetBeingTornDown(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-down") + seedRestoreSnapshot(t, st, "src-down", "snap-down", []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + csiPrepareRestoreTarget(t, base, "src-down", "snap-down", "pvc-down") + seedTerminatingReplica(t, st, "pvc-down", "node-a") + + if code := restoreOnce(t, base, "src-down", "snap-down", map[string]any{ + "to_resource": "pvc-down", "nodes": []string{"node-a"}, + }); code != http.StatusConflict { + t.Errorf("restore into a target being torn down = %d, want 409", code) + } + + assertTargetUntouched(t, st, "pvc-down") +} + +// The post-write group guard refused a prepared target parented to a group +// that is gone only after the marker was on it. +func TestSnapshotRestoreLeavesAPreparedTargetUnmarkedWhenItsGroupIsGone(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-nogrp") + seedRestoreSnapshot(t, st, "src-nogrp", "snap-nogrp", []string{"node-a"}) + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-nogrp", ResourceGroupName: "grp-gone", + }); err != nil { + t.Fatalf("seed the target: %v", err) + } + + if err := st.VolumeDefinitions().Create(ctx, "pvc-nogrp", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the target's volume: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := restoreOnce(t, base, "src-nogrp", "snap-nogrp", map[string]any{ + "to_resource": "pvc-nogrp", "nodes": []string{"node-a"}, + }); code != http.StatusConflict { + t.Errorf("restore into a target whose group is gone = %d, want 409", code) + } + + assertTargetUntouched(t, st, "pvc-nogrp") +} + +// The same stack is no difference, whatever the order or spelling. +func TestSnapshotRestoreTakesAPreparedTargetOfTheSourcesLayerStack(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedCloneSourceWithStack(t, st, "src-same", []string{"DRBD", "LUKS", "STORAGE"}) + seedRestoreSnapshot(t, st, "src-same", "snap-same", []string{"node-a"}) + + base, stop := startServerWithPassphrase(t, st) + defer stop() + + csiPrepareRestoreTargetWithLayers(t, base, "src-same", "snap-same", "pvc-same", []string{"drbd", "luks", "storage"}) + + if code := restoreOnce(t, base, "src-same", "snap-same", map[string]any{ + "to_resource": "pvc-same", "nodes": []string{"node-a"}, + }); code != http.StatusCreated { + t.Errorf("restore into a target of the source's own stack = %d, want 201", code) + } +} + +// A source stored without a stack has the default, not no layers, so a target +// that names the default explicitly is the same stack. +func TestSnapshotRestoreTakesAnExplicitDefaultStackOverAnImplicitOne(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedCloneSourceWithStack(t, st, "src-implicit", nil) + seedRestoreSnapshot(t, st, "src-implicit", "snap-implicit", []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + csiPrepareRestoreTargetWithLayers(t, base, "src-implicit", "snap-implicit", "pvc-explicit", + apiv1.DefaultLayerStack()) + + if code := restoreOnce(t, base, "src-implicit", "snap-implicit", map[string]any{ + "to_resource": "pvc-explicit", "nodes": []string{"node-a"}, + }); code != http.StatusCreated { + t.Errorf("restore into an explicit default stack over an implicit one = %d, want 201", code) + } +} diff --git a/pkg/rest/snapshot_restore_replay_test.go b/pkg/rest/snapshot_restore_replay_test.go new file mode 100644 index 00000000..51526942 --- /dev/null +++ b/pkg/rest/snapshot_restore_replay_test.go @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "encoding/json" + "net/http" + "slices" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +func seedRestoreSnapshot(t *testing.T, st store.Store, srcName, snapName string, nodes []string) { + t.Helper() + + if err := st.Snapshots().Create(t.Context(), &apiv1.Snapshot{ + Name: snapName, + ResourceName: srcName, + Nodes: nodes, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + }); err != nil { + t.Fatalf("seed snapshot %s: %v", snapName, err) + } +} + +func restoreOnce(t *testing.T, base, src, snap string, body map[string]any) int { + t.Helper() + + raw, err := json.Marshal(body) + if err != nil { + t.Fatalf("marshal restore body: %v", err) + } + + resp := httpPost(t, base+"/v1/resource-definitions/"+src+"/snapshot-restore-resource/"+snap, raw) + _ = resp.Body.Close() + + return resp.StatusCode +} + +// The restore replay resumed on the marker alone, and placement re-created a +// replica on every requested node that no longer had one, including a node the +// operator had emptied after the restore finished. +func TestSnapshotRestoreReplayLeavesAnEmptiedNodeAlone(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedTwoNodeCloneSource(t, st, "src-rr") + seedRestoreSnapshot(t, st, "src-rr", "snap-rr", []string{"node-a", "node-b"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + body := map[string]any{"to_resource": "dst-rr", "node_names": []string{"node-a", "node-b"}} + + if code := restoreOnce(t, base, "src-rr", "snap-rr", body); code != http.StatusCreated { + t.Fatalf("first restore = %d, want 201", code) + } + + if err := st.Resources().Delete(ctx, "dst-rr", "node-a"); err != nil { + t.Fatalf("empty node-a: %v", err) + } + + if code := restoreOnce(t, base, "src-rr", "snap-rr", body); code != http.StatusCreated { + t.Fatalf("replay of the finished restore = %d, want 201", code) + } + + after, err := st.Resources().ListByDefinition(ctx, "dst-rr") + if err != nil { + t.Fatalf("list the restored replicas: %v", err) + } + + if names := nodeNamesOf(after); !slices.Equal(names, []string{"node-b"}) { + t.Errorf("replicas after the replay = %v, want [node-b]: the replay re-stamped the emptied node", names) + } +} + +// A bare restore places nothing, so a replica is not what makes it finished. A +// volume added to it afterwards is its own, and the replay stays a replay. +func TestSnapshotRestoreReplayOfABareRestoreWithAnAddedVolume(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-rb") + seedRestoreSnapshot(t, st, "src-rb", "snap-rb", []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + body := map[string]any{"to_resource": "dst-rb"} + + if code := restoreOnce(t, base, "src-rb", "snap-rb", body); code != http.StatusCreated { + t.Fatalf("first restore = %d, want 201", code) + } + + if err := st.VolumeDefinitions().Create(ctx, "dst-rb", + &apiv1.VolumeDefinition{VolumeNumber: 1, SizeKib: 32 * 1024}); err != nil { + t.Fatalf("add a volume to the restored definition: %v", err) + } + + if code := restoreOnce(t, base, "src-rb", "snap-rb", body); code != http.StatusCreated { + t.Errorf("replay of a bare restore that gained a volume = %d, want 201", code) + } +} + +// With the replica requirement gone, an explicit-node restore whose first +// attempt hydrated the volumes and died before placing anything reads as +// finished, and the replay reports success over a target that exists on no +// node. +func TestSnapshotRestoreReplayPlacesWhenTheFirstAttemptPlacedNothing(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-np") + seedRestoreSnapshot(t, st, "src-np", "snap-np", []string{"node-a"}) + + // The leftover a first attempt left: the marker, the snapshot's volumes, + // and no Resource at all. + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-np", + Props: map[string]string{ + restoreFromSnapshotKey: restoreMarker("src-np", "snap-np"), + }, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + if err := st.VolumeDefinitions().Create(ctx, "dst-np", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the hydrated volume: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := restoreOnce(t, base, "src-np", "snap-np", map[string]any{ + "to_resource": "dst-np", "node_names": []string{"node-a"}, + }); code != http.StatusCreated { + t.Fatalf("retry = %d, want 201", code) + } + + placed, err := st.Resources().ListByDefinition(ctx, "dst-np") + if err != nil { + t.Fatalf("list the restored replicas: %v", err) + } + + if len(placed) == 0 { + t.Error("the retry reported the restore done over a definition with no replica") + } +} + +// Every existing fixture for the DELETE refusal on the restore door seeds a +// leftover with no volumes, so a deeper check gives the same 409 and the term +// never decides. With volumes and a replica, the replay would answer 201 over +// a definition being deleted. +func TestSnapshotRestoreReplayRefusesAFinishedLeftoverBeingDeleted(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-del") + seedRestoreSnapshot(t, st, "src-del", "snap-del", []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + body := map[string]any{"to_resource": "dst-del", "node_names": []string{"node-a"}} + + if code := restoreOnce(t, base, "src-del", "snap-del", body); code != http.StatusCreated { + t.Fatalf("first restore = %d, want 201", code) + } + + updateSource(t, st, "dst-del", func(rd *apiv1.ResourceDefinition) { + rd.Flags = append(rd.Flags, rdFlagDelete) + }) + + if code := restoreOnce(t, base, "src-del", "snap-del", body); code != http.StatusConflict { + t.Errorf("replay over a finished restore being deleted = %d, want 409", code) + } +} diff --git a/pkg/rest/snapshot_restore_resume_gates_test.go b/pkg/rest/snapshot_restore_resume_gates_test.go new file mode 100644 index 00000000..2db51228 --- /dev/null +++ b/pkg/rest/snapshot_restore_resume_gates_test.go @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "context" + "net/http" + "strings" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// The first call of a restore into a target the caller prepared is that +// restore's first run. A prepared target restored without nodes needs no +// replica to be finished, so it reached the finished-replay answer, which said +// "completed on retry" to a call that retried nothing. +func TestSnapshotRestoreIntoAPreparedTargetWithoutNodesIsNotARetry(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-pbare") + seedRestoreSnapshot(t, st, "src-pbare", "snap-pbare", []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + csiPrepareRestoreTarget(t, base, "src-pbare", "snap-pbare", "pvc-pbare") + + code, rc := restoreAnswer(t, base, "src-pbare", "snap-pbare", map[string]any{"to_resource": "pvc-pbare"}) + if code != http.StatusCreated { + t.Fatalf("restore into the prepared target = %d (%s), want 201", code, rc.Message) + } + + if strings.Contains(rc.Message, "on retry") { + t.Errorf("first restore into a prepared target answered %q", rc.Message) + } + + code, rc = restoreAnswer(t, base, "src-pbare", "snap-pbare", map[string]any{"to_resource": "pvc-pbare"}) + if code != http.StatusCreated || !strings.Contains(rc.Message, "on retry") { + t.Errorf("the replay answered %d %q, want 201 on retry", code, rc.Message) + } +} + +// A resume over a leftover parented to a group that is gone hydrated and +// placed into it first, and only the post-write guard refused it then, which +// does not roll back a definition the request adopted. The refusal has to come +// before anything is written. +func TestSnapshotRestoreResumeRefusesAnOrphanedLeftoverBeforeWriting(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-orph") + seedRestoreSnapshot(t, st, "src-orph", "snap-orph", []string{"node-a"}) + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-orph", + ResourceGroupName: "grp-gone", + Props: map[string]string{restoreFromSnapshotKey: restoreMarker("src-orph", "snap-orph")}, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + code, rc := restoreAnswer(t, base, "src-orph", "snap-orph", map[string]any{ + "to_resource": "dst-orph", "nodes": []string{"node-a"}, + }) + if code != http.StatusConflict { + t.Errorf("resume over a leftover whose group is gone = %d (%s), want 409", code, rc.Message) + } + + if vds, _ := st.VolumeDefinitions().List(ctx, "dst-orph"); len(vds) > 0 { + t.Errorf("the refused resume hydrated %d volume(s) into the leftover", len(vds)) + } + + if replicas, _ := st.Resources().ListByDefinition(ctx, "dst-orph"); len(replicas) > 0 { + t.Errorf("the refused resume placed %d replica(s) on the leftover", len(replicas)) + } +} + +// cacheMissesTheDelete serves a replica from the cache without the DELETE flag +// the API server already holds, the way an informer does right after a delete +// began, and answers GetUncached from the backend. +type cacheMissesTheDelete struct { + store.Store +} + +type cacheMissesTheDeleteResources struct { + store.ResourceStore +} + +func (c cacheMissesTheDelete) Resources() store.ResourceStore { + return cacheMissesTheDeleteResources{c.Store.Resources()} +} + +func (r cacheMissesTheDeleteResources) Get(ctx context.Context, rdName, node string) (apiv1.Resource, error) { + res, err := r.ResourceStore.Get(ctx, rdName, node) + res.Flags = nil + + return res, err //nolint:wrapcheck // pass-through test double +} + +func (r cacheMissesTheDeleteResources) GetUncached(ctx context.Context, rdName, node string) (apiv1.Resource, error) { + return r.ResourceStore.Get(ctx, rdName, node) //nolint:wrapcheck // pass-through test double +} + +// The replica a stamp collided with was read through the cache, which right +// after a delete began still served it live, and the stamp counted a replica +// that was going away as placed. +func TestRDCloneReadsACollidingReplicaPastTheCache(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, backend, "src-lagdel") + + if err := backend.Resources().Create(ctx, &apiv1.Resource{ + Name: "dst-lagdel", NodeName: "node-a", Flags: []string{apiv1.ResourceFlagDelete}, + }); err != nil { + t.Fatalf("seed the terminating replica: %v", err) + } + + base, stop := startServerWithStore(t, cacheMissesTheDelete{backend}) + defer stop() + + if code := cloneOnce(t, base, "src-lagdel", "dst-lagdel", nil); code == http.StatusCreated { + t.Errorf("clone over a replica the cache still served live = 201") + } +} diff --git a/pkg/rest/snapshot_restore_test.go b/pkg/rest/snapshot_restore_test.go index 982cb201..40d0c9fd 100644 --- a/pkg/rest/snapshot_restore_test.go +++ b/pkg/rest/snapshot_restore_test.go @@ -19,6 +19,7 @@ limitations under the License. package rest import ( + "context" "encoding/json" "net/http" "testing" @@ -1106,3 +1107,139 @@ func TestSnapshotRestoreVolumeDefinitionConflict_G3b(t *testing.T) { t.Errorf("target VD 0 size: got %d, want untouched 2048 MiB", got.SizeKib) } } + +// volumeListLags hides the target's volumes from List, the way a pre-check's +// list taken a moment before a concurrent restore wrote them does. +type volumeListLags struct { + store.VolumeDefinitionStore + + target string +} + +func (v volumeListLags) List(ctx context.Context, rdName string) ([]apiv1.VolumeDefinition, error) { + if rdName == v.target { + return nil, nil + } + + return v.VolumeDefinitionStore.List(ctx, rdName) //nolint:wrapcheck // pass-through test double +} + +type volumeListLagsStore struct { + store.Store + + target string +} + +func (s volumeListLagsStore) VolumeDefinitions() store.VolumeDefinitionStore { + return volumeListLags{VolumeDefinitionStore: s.Store.VolumeDefinitions(), target: s.target} +} + +// A volume that appeared between the volume-definition restore's collision +// check and its create is refused even at the snapshot's size, as on main. A +// concurrent CLI restore of the same snapshot that wrote it can still fail on +// a later volume and unwind it, so a 200 over it reports a layout about to go. +func TestSnapshotRestoreVolumeDefinitionRefusesAVolumeThatAppearedAtTheSnapshotsSize(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "pvc-src-race"}); err != nil { + t.Fatalf("seed source RD: %v", err) + } + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "snap-race", ResourceName: "pvc-src-race", + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 1024 * 1024}}, + }); err != nil { + t.Fatalf("seed snap: %v", err) + } + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "pvc-tgt-race"}); err != nil { + t.Fatalf("seed target RD: %v", err) + } + + if err := st.VolumeDefinitions().Create(ctx, "pvc-tgt-race", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 1024 * 1024}); err != nil { + t.Fatalf("seed the volume the concurrent restore wrote: %v", err) + } + + base, stop := startServerWithStore(t, volumeListLagsStore{Store: st, target: "pvc-tgt-race"}) + defer stop() + + body, _ := json.Marshal(snapshotRestoreRequest{ToResource: "pvc-tgt-race"}) + + resp := httpPost(t, + base+"/v1/resource-definitions/pvc-src-race/snapshot-restore-volume-definition/snap-race", body) + defer func() { _ = resp.Body.Close() }() + + var rcs []apiv1.APICallRc + if err := json.NewDecoder(resp.Body).Decode(&rcs); err != nil || len(rcs) == 0 { + t.Fatalf("decode the answer (status %d): %v", resp.StatusCode, err) + } + + if resp.StatusCode != http.StatusConflict { + t.Errorf("volume-definition restore over a volume another restore wrote = %d, want 409", resp.StatusCode) + } + + for _, rc := range rcs { + if rc.RetCode&apiCallRcFailExistsVlmDfn == apiCallRcFailExistsVlmDfn { + t.Errorf("refusal ret_code %#x carries the FAIL_EXISTS_VLM_DFN band, which reads as already restored", rc.RetCode) + } + } +} + +// The volume-definition restore refuses a definition another restore has +// marked as its own, before writing anything: that restore's rollback would +// take the definition with the volumes this answered 200 for. +func TestSnapshotRestoreVolumeDefinitionRefusesATargetUnderARestoreMarker(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "pvc-src-mk"}); err != nil { + t.Fatalf("seed source RD: %v", err) + } + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "snap-mk", ResourceName: "pvc-src-mk", + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 1024 * 1024}}, + }); err != nil { + t.Fatalf("seed snap: %v", err) + } + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-tgt-mk", Props: map[string]string{restoreFromSnapshotKey: restoreMarker("pvc-src-mk", "snap-mk")}, + }); err != nil { + t.Fatalf("seed the marked target: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(snapshotRestoreRequest{ToResource: "pvc-tgt-mk"}) + + resp := httpPost(t, + base+"/v1/resource-definitions/pvc-src-mk/snapshot-restore-volume-definition/snap-mk", body) + defer func() { _ = resp.Body.Close() }() + + var rcs []apiv1.APICallRc + if err := json.NewDecoder(resp.Body).Decode(&rcs); err != nil || len(rcs) == 0 { + t.Fatalf("decode the answer (status %d): %v", resp.StatusCode, err) + } + + if resp.StatusCode != http.StatusConflict { + t.Errorf("volume-definition restore into a marked target = %d, want 409", resp.StatusCode) + } + + for _, rc := range rcs { + if rc.RetCode&(apiCallRcFailExistsRscDfn|apiCallRcFailExistsVlmDfn) != 0 { + t.Errorf("refusal ret_code %#x carries a FAIL_EXISTS band", rc.RetCode) + } + } + + if vds, _ := st.VolumeDefinitions().List(ctx, "pvc-tgt-mk"); len(vds) != 0 { + t.Errorf("the refused restore wrote %d volume(s)", len(vds)) + } +} diff --git a/pkg/rest/snapshot_restore_volume_definition_test.go b/pkg/rest/snapshot_restore_volume_definition_test.go new file mode 100644 index 00000000..02e0099f --- /dev/null +++ b/pkg/rest/snapshot_restore_volume_definition_test.go @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "encoding/json" + "net/http" + "strings" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// The volume-definition restore fetched the target definition and threw it +// away. Its siblings refuse a target carrying DELETE because finishing one +// races the tear-down reaping what it writes, and volumes hydrated here are +// precisely that. +func TestSnapshotRestoreVolumeDefinitionRefusesADyingTarget(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedRestoreSource(ctx, t, st) + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-vd-dying", + Flags: []string{rdFlagDelete}, + }); err != nil { + t.Fatalf("seed the dying target: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(map[string]string{"to_resource": "pvc-vd-dying"}) + + resp := httpPost(t, + base+"/v1/resource-definitions/pvc-src/snapshot-restore-volume-definition/snap-1", body) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409 — that definition is being deleted", resp.StatusCode) + } + + if msg := decodeRCs(t, resp)[0].Message; !strings.Contains(msg, "being deleted") { + t.Errorf("message = %q, want it to name the deletion", msg) + } + + vds, err := st.VolumeDefinitions().List(ctx, "pvc-vd-dying") + if err != nil { + t.Fatalf("list the target's volumes: %v", err) + } + + if len(vds) != 0 { + t.Errorf("hydrated %d volume(s) into a definition being deleted", len(vds)) + } +} + +// The volume-definition restore's collision guard has two halves: a pre-check +// that LISTs the target's volumes, and the hydrate that CREATEs them. The +// pre-check's own comment waves a request through when that list cannot be +// read, on the stated grounds that "the downstream hydrate Create still +// guards" — so a blanket AlreadyExists tolerance in the hydrate turns the +// endpoint into a 200 reporting a layout it never wrote. +// +// The same hole opens with no read error at all, since one call LISTs and the +// other CREATEs: a volume appearing between the two arrives at the hydrate. +func TestSnapshotRestoreVolumeDefinitionRefusesAVolumeAtADifferentSize(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + + if err := st.ResourceDefinitions().Create(ctx, + &apiv1.ResourceDefinition{Name: "vd-src"}); err != nil { + t.Fatalf("seed the source: %v", err) + } + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "snap-vd", + ResourceName: "vd-src", + Nodes: []string{"n1"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + }); err != nil { + t.Fatalf("seed the snapshot: %v", err) + } + + if err := st.ResourceDefinitions().Create(ctx, + &apiv1.ResourceDefinition{Name: "vd-dst"}); err != nil { + t.Fatalf("seed the target: %v", err) + } + + // A volume already under that number, recording a different size: not the + // one this snapshot would write. + if err := st.VolumeDefinitions().Create(ctx, "vd-dst", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 8 * 1024}); err != nil { + t.Fatalf("seed the colliding volume: %v", err) + } + + // The pre-check LISTs and the hydrate CREATEs, and the pre-check's own + // comment waves a request through when that list cannot be read, on the + // grounds that the hydrate still guards. Blinding the list is how this + // test asks whether that is true — and it is also the no-error case, where + // a volume simply appears between the two calls. + base, stop := startServerWithStore(t, blindVolumeListStore{st}) + defer stop() + + body := []byte(`{"to_resource":"vd-dst"}`) + + resp := httpPost(t, + base+"/v1/resource-definitions/vd-src/snapshot-restore-volume-definition/snap-vd", body) + _ = resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + t.Fatalf("status = 200 — the operator was told the layout was restored, and it was not") + } + + vd, err := st.VolumeDefinitions().Get(ctx, "vd-dst", 0) + if err != nil { + t.Fatalf("read the target's volume: %v", err) + } + + if vd.SizeKib != 8*1024 { + t.Errorf("volume 0 = %d KiB; the refused restore rewrote it", vd.SizeKib) + } +} + +// The ownership term is what separates a restore's own expanded volume from a +// larger volume on a definition the volume-definition restore does not own. +// The existing size fixture seeds a smaller volume, so it never reached it. +func TestSnapshotRestoreVolumeDefinitionRefusesALargerVolumeItDidNotWrite(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "vdl-src"}); err != nil { + t.Fatalf("seed the source: %v", err) + } + + seedRestoreSnapshot(t, st, "vdl-src", "snap-vdl", []string{"n1"}) + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "vdl-dst"}); err != nil { + t.Fatalf("seed the target: %v", err) + } + + if err := st.VolumeDefinitions().Create(ctx, "vdl-dst", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 256 * 1024}); err != nil { + t.Fatalf("seed the larger volume: %v", err) + } + + base, stop := startServerWithStore(t, blindVolumeListStore{st}) + defer stop() + + resp := httpPost(t, base+"/v1/resource-definitions/vdl-src/snapshot-restore-volume-definition/snap-vdl", + []byte(`{"to_resource":"vdl-dst"}`)) + _ = resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + t.Error("status = 200 over a larger volume the restore did not write") + } +} diff --git a/pkg/rest/snapshot_restore_without_snapshot_test.go b/pkg/rest/snapshot_restore_without_snapshot_test.go new file mode 100644 index 00000000..665627ef --- /dev/null +++ b/pkg/rest/snapshot_restore_without_snapshot_test.go @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "net/http" + "strings" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// Deleting the snapshot once the restore finished is ordinary cleanup, and a +// retry after it answered 404: the replay was reached only through a snapshot +// read. A finished target is judged against the volumes it recorded. +func TestSnapshotRestoreReplayOverAFinishedTargetSurvivesTheSnapshot(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-gone") + seedRestoreSnapshot(t, st, "src-gone", "snap-gone", []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + body := map[string]any{"to_resource": "dst-gone", "node_names": []string{"node-a"}} + if code := restoreOnce(t, base, "src-gone", "snap-gone", body); code != http.StatusCreated { + t.Fatalf("first restore = %d, want 201", code) + } + + if err := st.Snapshots().Delete(ctx, "src-gone", "snap-gone"); err != nil { + t.Fatalf("delete the snapshot: %v", err) + } + + before, _ := st.Resources().ListByDefinition(ctx, "dst-gone") + + code, rc := restoreAnswer(t, base, "src-gone", "snap-gone", body) + if code != http.StatusCreated { + t.Fatalf("replay with the snapshot gone = %d %q, want 201", code, rc.Message) + } + + after, _ := st.Resources().ListByDefinition(ctx, "dst-gone") + if len(after) != len(before) { + t.Errorf("the replay placed replicas: %d before, %d after", len(before), len(after)) + } +} + +// An unfinished target cannot be finished from a snapshot that is gone, and +// the refusal names the way out. +func TestSnapshotRestoreRefusesAnUnfinishedTargetWhoseSnapshotIsGone(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-half") + + snap := apiv1.Snapshot{ + Name: "snap-half", ResourceName: "src-half", + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + } + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-half", Props: store.WithRestoreMarker(nil, &snap), + }); err != nil { + t.Fatalf("seed the unfinished target: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + code, rc := restoreAnswer(t, base, "src-half", "snap-half", map[string]any{"to_resource": "dst-half"}) + if code != http.StatusNotFound { + t.Fatalf("restore over an unfinished target with the snapshot gone = %d %q, want 404", code, rc.Message) + } + + if !strings.Contains(rc.Correc, "restore it from another snapshot") { + t.Errorf("correction %q does not name the way out", rc.Correc) + } + + if vds, _ := st.VolumeDefinitions().List(ctx, "dst-half"); len(vds) != 0 { + t.Errorf("the refused restore hydrated %d volume(s)", len(vds)) + } +} + +// A target restored before the volumes were recorded keeps the 404 it got: +// there is nothing to judge it against. +func TestSnapshotRestoreWithoutTheSnapshotKeepsALegacyTargetA404(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-legacy") + seedRestoreLeftover(t, st, "src-legacy", "snap-legacy", "dst-legacy", nil) + + // Whole by its own volumes: judged without a record it would read as a + // finished restore and answer 201 over volumes nothing vouches for. + if err := st.VolumeDefinitions().Create(ctx, "dst-legacy", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the legacy volume: %v", err) + } + + if err := st.Resources().Create(ctx, &apiv1.Resource{Name: "dst-legacy", NodeName: "node-a"}); err != nil { + t.Fatalf("seed the legacy replica: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code, rc := restoreAnswer(t, base, "src-legacy", "snap-legacy", + map[string]any{"to_resource": "dst-legacy"}); code != http.StatusNotFound { + t.Errorf("restore over a legacy target with the snapshot gone = %d %q, want 404", code, rc.Message) + } +} + +// seedFinishedTargetWithoutSnapshot seeds a target restored from snapName, +// finished (its volumes and a live replica), with the snapshot already gone. +func seedFinishedTargetWithoutSnapshot(t *testing.T, st store.Store, src, snapName, target string, flags []string) { + t.Helper() + + ctx := t.Context() + snap := apiv1.Snapshot{ + Name: snapName, ResourceName: src, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + } + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: target, Props: store.WithRestoreMarker(nil, &snap), Flags: flags, + }); err != nil { + t.Fatalf("seed the target: %v", err) + } + + if err := st.VolumeDefinitions().Create(ctx, target, + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("seed the target's volume: %v", err) + } + + if err := st.Resources().Create(ctx, &apiv1.Resource{Name: target, NodeName: "node-a"}); err != nil { + t.Fatalf("seed the target's replica: %v", err) + } +} + +// A finished target being deleted is not a replay to report done, snapshot or +// no snapshot. +func TestSnapshotRestoreWithoutSnapshotRefusesATargetBeingDeleted(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-del") + seedFinishedTargetWithoutSnapshot(t, st, "src-del", "snap-del", "dst-del", []string{rdFlagDelete}) + + base, stop := startServerWithStore(t, st) + defer stop() + + code, rc := restoreAnswer(t, base, "src-del", "snap-del", map[string]any{"to_resource": "dst-del"}) + if code != http.StatusConflict { + t.Fatalf("replay over a target being deleted = %d %q, want 409", code, rc.Message) + } + + if rc.RetCode&apiCallRcFailExistsRscDfn != apiCallRcFailExistsRscDfn { + t.Errorf("refusal ret_code %#x carries no FAIL_EXISTS_RSC_DFN band", rc.RetCode) + } +} + +// A target restored from another snapshot is not this restore's replay: it +// would report a restore that never happened as done. +func TestSnapshotRestoreWithoutSnapshotIgnoresATargetOfAnotherSnapshot(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-other") + seedFinishedTargetWithoutSnapshot(t, st, "src-other", "snap-b", "dst-other", nil) + + base, stop := startServerWithStore(t, st) + defer stop() + + code, rc := restoreAnswer(t, base, "src-other", "snap-a", map[string]any{"to_resource": "dst-other"}) + if code != http.StatusNotFound { + t.Errorf("restore of a gone snapshot over another snapshot's target = %d %q, want 404", code, rc.Message) + } +} + +// Whether the target is a finished replay cannot be told when it cannot be +// read, and the snapshot's 404 would read as permanent. +func TestSnapshotRestoreWithoutSnapshotAnswersAFailedTargetReadAsRetryable(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-rd") + seedFinishedTargetWithoutSnapshot(t, backend, "src-rd", "snap-rd", "dst-rd", nil) + + base, stop := startServerWithStore(t, faultyStore{backend, storeFaults{ + target: "dst-rd", rdGetUncached: errStoreFault, + }}) + defer stop() + + code, rc := restoreAnswer(t, base, "src-rd", "snap-rd", map[string]any{"to_resource": "dst-rd"}) + if code != http.StatusInternalServerError { + t.Fatalf("replay whose target cannot be read = %d %q, want 500", code, rc.Message) + } + + if !strings.Contains(rc.Correc, "retry") { + t.Errorf("correction %q does not say to retry", rc.Correc) + } +} + +// A bare restore is finished without a replica only while the snapshot its +// placement would restore from exists. Once that is gone a shell with no +// replica holds its data nowhere, and the replay answered 201 over it while +// the CLI, judging the same definition, refused it. One placed since keeps its +// 201. +func TestSnapshotRestoreReplayOfABareRestoreNeedsAReplicaOnceTheSnapshotIsGone(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + replica bool + }{ + {name: "no-replica"}, + {name: "placed-since", replica: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-bare") + seedRestoreSnapshot(t, st, "src-bare", "snap-bare", []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + body := map[string]any{"to_resource": "dst-bare"} + if code := restoreOnce(t, base, "src-bare", "snap-bare", body); code != http.StatusCreated { + t.Fatalf("bare restore = %d, want 201", code) + } + + if tc.replica { + if err := st.Resources().Create(ctx, &apiv1.Resource{Name: "dst-bare", NodeName: "node-a"}); err != nil { + t.Fatalf("place a replica: %v", err) + } + } + + if err := st.Snapshots().Delete(ctx, "src-bare", "snap-bare"); err != nil { + t.Fatalf("delete the snapshot: %v", err) + } + + code, rc := restoreAnswer(t, base, "src-bare", "snap-bare", body) + + if tc.replica { + if code != http.StatusCreated { + t.Errorf("replay of a bare restore placed since = %d %q, want 201", code, rc.Message) + } + + return + } + + if code != http.StatusNotFound || !strings.Contains(rc.Correc, "restore it from another snapshot") { + t.Errorf("replay of a bare restore with no replica and no snapshot = %d %q / %q, want 404 naming the way out", + code, rc.Message, rc.Correc) + } + }) + } +} + +// A restore being torn down once its snapshot is gone cannot be restored +// again after the tear-down: there is nothing left to restore it from, so the +// refusal says to restore it from another snapshot rather than to wait and +// retry into the snapshot-gone refusal. +func TestSnapshotRestoreTearDownWithTheSnapshotGoneNamesAnotherSnapshot(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-tdown") + seedRestoreSnapshot(t, st, "src-tdown", "snap-tdown", []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + body := map[string]any{"to_resource": "dst-tdown"} + if code := restoreOnce(t, base, "src-tdown", "snap-tdown", body); code != http.StatusCreated { + t.Fatalf("bare restore = %d, want 201", code) + } + + seedTerminatingReplica(t, st, "dst-tdown", "node-a") + + if err := st.Snapshots().Delete(ctx, "src-tdown", "snap-tdown"); err != nil { + t.Fatalf("delete the snapshot: %v", err) + } + + code, rc := restoreAnswer(t, base, "src-tdown", "snap-tdown", body) + if code != http.StatusConflict || !strings.Contains(rc.Correc, "restore it from another snapshot") { + t.Errorf("replay over a tear-down with the snapshot gone = %d %q / %q, want 409 naming another snapshot", + code, rc.Message, rc.Correc) + } +} + +// A leftover holding a volume the snapshot never recorded is refused, and once +// the snapshot is gone the refusal does not say to delete it and "restore +// again": that restore would meet nothing to restore from. +func TestSnapshotRestoreForeignLeftoverWithTheSnapshotGoneNamesAnotherSnapshot(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-fgone") + seedRestoreSnapshot(t, st, "src-fgone", "snap-fgone", []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + body := map[string]any{"to_resource": "dst-fgone"} + if code := restoreOnce(t, base, "src-fgone", "snap-fgone", body); code != http.StatusCreated { + t.Fatalf("bare restore = %d, want 201", code) + } + + if err := st.VolumeDefinitions().Create(ctx, "dst-fgone", + &apiv1.VolumeDefinition{VolumeNumber: 1, SizeKib: 64 * 1024}); err != nil { + t.Fatalf("add a volume the snapshot never recorded: %v", err) + } + + if err := st.Snapshots().Delete(ctx, "src-fgone", "snap-fgone"); err != nil { + t.Fatalf("delete the snapshot: %v", err) + } + + code, rc := restoreAnswer(t, base, "src-fgone", "snap-fgone", body) + if code != http.StatusConflict || !strings.Contains(rc.Correc, "another snapshot") || + strings.Contains(rc.Correc, "restore again") { + t.Errorf("replay over a foreign leftover with the snapshot gone = %d %q / %q, want 409 naming another snapshot", + code, rc.Message, rc.Correc) + } +} diff --git a/pkg/rest/sp_namespace_spelling_test.go b/pkg/rest/sp_namespace_spelling_test.go new file mode 100644 index 00000000..7f013c53 --- /dev/null +++ b/pkg/rest/sp_namespace_spelling_test.go @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "encoding/json" + "net/http" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// Every spelling of the StorDriver namespace that the deleter honours is one +// the backing-identity guard refuses, on both pool doors: a spelling the guard +// read another way would strip a live pool's identity, and the same guard then +// refuses to put it back. +func TestPoolDoorsRefuseEverySpellingOfTheDriverNamespace(t *testing.T) { + t.Parallel() + + for _, ns := range []string{"StorDriver", "StorDriver/", "StorDriver//"} { + t.Run("pool "+ns, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + + if err := st.Nodes().Create(ctx, &apiv1.Node{Name: "n1", Type: apiv1.NodeTypeSatellite}); err != nil { + t.Fatalf("seed node: %v", err) + } + + if err := st.StoragePools().Create(ctx, &apiv1.StoragePool{ + NodeName: "n1", StoragePoolName: "zfs-thin", ProviderKind: apiv1.StoragePoolKindZFSThin, + Props: map[string]string{"StorDriver/ZPoolThin": "blockstor-zfs"}, + }); err != nil { + t.Fatalf("seed pool: %v", err) + } + + code := putNamespaceDelete(t, st, "/v1/nodes/n1/storage-pools/zfs-thin", ns) + if code != http.StatusBadRequest { + t.Errorf("delete_namespaces=[%q] on a pool = %d, want 400", ns, code) + } + + sp, err := st.StoragePools().Get(ctx, "n1", "zfs-thin") + if err != nil { + t.Fatalf("read the pool: %v", err) + } + + if sp.Props["StorDriver/ZPoolThin"] != "blockstor-zfs" { + t.Errorf("delete_namespaces=[%q] stripped the pool's backing identity", ns) + } + }) + + t.Run("definition "+ns, func(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + + if err := st.StoragePoolDefinitions().Create(ctx, &store.StoragePoolDefinition{ + Name: "zfs-thin", Props: map[string]string{"StorDriver/ZPoolThin": "blockstor-zfs"}, + }); err != nil { + t.Fatalf("seed definition: %v", err) + } + + code := putNamespaceDelete(t, st, "/v1/storage-pool-definitions/zfs-thin", ns) + if code != http.StatusBadRequest { + t.Errorf("delete_namespaces=[%q] on a pool definition = %d, want 400", ns, code) + } + + def, err := st.StoragePoolDefinitions().Get(ctx, "zfs-thin") + if err != nil { + t.Fatalf("read the definition: %v", err) + } + + if def.Props["StorDriver/ZPoolThin"] != "blockstor-zfs" { + t.Errorf("delete_namespaces=[%q] stripped the definition's backing identity", ns) + } + }) + } +} + +func putNamespaceDelete(t *testing.T, st store.Store, path, ns string) int { + t.Helper() + + base, stop := startServerWithStore(t, st) + defer stop() + + body, err := json.Marshal(apiv1.GenericPropsModify{DeleteNamespace: []string{ns}}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + resp := httpPut(t, base+path, body) + _ = resp.Body.Close() + + return resp.StatusCode +} diff --git a/pkg/rest/spawn.go b/pkg/rest/spawn.go index 89a6889b..51c8cd8f 100644 --- a/pkg/rest/spawn.go +++ b/pkg/rest/spawn.go @@ -238,9 +238,14 @@ func buildSpawnedRD(req *apiv1.ResourceGroupSpawn, rgName string, rg *apiv1.Reso ResourceGroupName: rgName, } + // The group's props come through laundered: the props blockstor reads + // back off a definition must not be inherited, or a group carrying one, + // which linstor-csi fills from StorageClass parameters verbatim, would + // mark every definition it spawns as restoring from another resource's + // snapshot. if len(rg.Props) > 0 { rd.Props = make(map[string]string, len(rg.Props)) - maps.Copy(rd.Props, rg.Props) + maps.Copy(rd.Props, store.TravellingProps(rg.Props)) } // Sticky LayerStack inheritance (Bug 54). Mirrors handleRDCreate's diff --git a/pkg/rest/storage_pool_definitions.go b/pkg/rest/storage_pool_definitions.go index 3f81a665..d1e42569 100644 --- a/pkg/rest/storage_pool_definitions.go +++ b/pkg/rest/storage_pool_definitions.go @@ -230,15 +230,7 @@ func applySPDfnPropsPatch(def *store.StoragePoolDefinition, patch *apiv1.Generic // I1: empty override value deletes the key (set-property KEY ""). applyPropsModify(def.Props, patch.OverrideProps, patch.DeleteProps) - for _, ns := range patch.DeleteNamespace { - prefix := ns + "/" - - for k := range def.Props { - if strings.HasPrefix(k, prefix) { - delete(def.Props, k) - } - } - } + deletePropNamespaces(def.Props, patch.DeleteNamespace) } // handleStoragePoolDefinitionDelete serves DELETE /v1/storage-pool-definitions/{name}. diff --git a/pkg/rest/storage_pools.go b/pkg/rest/storage_pools.go index 20d30278..eb1eeb21 100644 --- a/pkg/rest/storage_pools.go +++ b/pkg/rest/storage_pools.go @@ -119,15 +119,7 @@ func (s *Server) handleNodeStoragePoolModify(w http.ResponseWriter, r *http.Requ // behaviour — the namespace separator is '/' and the prefix // is matched literally (no glob). An entry "Aux" drops every // "Aux/..." key. - for _, ns := range patch.DeleteNamespace { - prefix := ns + "/" - - for k := range sp.Props { - if strings.HasPrefix(k, prefix) { - delete(sp.Props, k) - } - } - } + deletePropNamespaces(sp.Props, patch.DeleteNamespace) return nil }) @@ -791,10 +783,11 @@ func immutableSPDriverProps() []string { // // - override_props: an explicit value for any immutable key. // - delete_props: an explicit key in the immutable set. -// - delete_namespaces: a prefix that, with the '/' separator -// appended, would catch any immutable key. The shared "StorDriver" -// prefix means deleting the namespace would wipe the pool's -// backing identity wholesale, so refuse that too. +// - delete_namespaces: a namespace that covers any immutable key, read +// the way the delete reads it (store.NamespaceCovers), so every +// spelling the delete honours is refused here too. The shared +// "StorDriver" namespace would wipe the pool's backing identity +// wholesale. // // Returns ("", false) when the patch is clean (no immutable keys // touched). The caller writes the typed-envelope error on refuse. @@ -823,10 +816,8 @@ func refuseSPDriverPropMutation(patch *apiv1.GenericPropsModify) (string, bool) // would otherwise nuke the backing identity wholesale. Flag the // whole namespace so the operator sees the actionable name. for _, ns := range patch.DeleteNamespace { - prefix := ns + "/" - for _, key := range keys { - if strings.HasPrefix(key, prefix) { + if store.NamespaceCovers(ns, key) { hit = append(hit, "delete_namespaces:"+ns) break diff --git a/pkg/rest/store_faults_test.go b/pkg/rest/store_faults_test.go new file mode 100644 index 00000000..c20a4024 --- /dev/null +++ b/pkg/rest/store_faults_test.go @@ -0,0 +1,742 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "context" + "errors" + "net/http" + "strings" + "sync/atomic" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +var errStoreFault = errors.New("injected store failure") + +// storeFaults fails or bends one store call each, for the definition named +// target. The volume and replica stores are wrapped without their uncached +// reads, so the code under test falls back to the methods failed here. +type storeFaults struct { + target string + + snapshotList error // Snapshots().ListByDefinitionUncached of any definition + volumeList error // VolumeDefinitions().List of target + volumeCollides bool // VolumeDefinitions().Create answers AlreadyExists + volumeGet error // VolumeDefinitions().Get of target + rdCollides bool // ResourceDefinitions().Create of target answers AlreadyExists + rdGetUncached error // ResourceDefinitions().GetUncached of target + rdGet error // ResourceDefinitions().Get of target + // rdGetUncachedFails, when set, limits rdGetUncached to that many reads; + // the ones after it go through. + rdGetUncachedFails *atomic.Int32 + // rdGetPasses and rdGetUncachedPasses, when set, let that many reads of + // the target through and fail the one after them alone, so a test can + // aim at a single read site. + rdGetPasses, rdGetUncachedPasses *atomic.Int32 + replicaCollide bool // Resources().Create of target answers AlreadyExists + replicaGet error // Resources().Get of target + replicaPatch error // Resources().PatchResourceSpec of target + groupGet error // ResourceGroups().Get of any group +} + +type faultyStore struct { + store.Store + + f storeFaults +} + +type faultySnapshots struct { + store.SnapshotStore + + f storeFaults +} + +func (s faultySnapshots) ListByDefinitionUncached(ctx context.Context, rdName string) ([]apiv1.Snapshot, error) { + if s.f.snapshotList != nil { + return nil, s.f.snapshotList + } + + return s.SnapshotStore.ListByDefinitionUncached(ctx, rdName) //nolint:wrapcheck // pass-through test double +} + +type faultyVolumes struct { + store.VolumeDefinitionStore + + f storeFaults +} + +func (v faultyVolumes) List(ctx context.Context, rdName string) ([]apiv1.VolumeDefinition, error) { + if rdName == v.f.target && v.f.volumeList != nil { + return nil, v.f.volumeList + } + + return v.VolumeDefinitionStore.List(ctx, rdName) //nolint:wrapcheck // pass-through test double +} + +func (v faultyVolumes) Create(ctx context.Context, rdName string, vd *apiv1.VolumeDefinition) error { + if rdName == v.f.target && v.f.volumeCollides { + return store.ErrAlreadyExists + } + + return v.VolumeDefinitionStore.Create(ctx, rdName, vd) //nolint:wrapcheck // pass-through test double +} + +func (v faultyVolumes) Get(ctx context.Context, rdName string, number int32) (apiv1.VolumeDefinition, error) { + if rdName == v.f.target && v.f.volumeGet != nil { + return apiv1.VolumeDefinition{}, v.f.volumeGet + } + + return v.VolumeDefinitionStore.Get(ctx, rdName, number) //nolint:wrapcheck // pass-through test double +} + +type faultyDefinitions struct { + store.ResourceDefinitionStore + + f storeFaults +} + +func (d faultyDefinitions) Create(ctx context.Context, rd *apiv1.ResourceDefinition) error { + if rd.Name == d.f.target && d.f.rdCollides { + return store.ErrAlreadyExists + } + + return d.ResourceDefinitionStore.Create(ctx, rd) //nolint:wrapcheck // pass-through test double +} + +func (d faultyDefinitions) Get(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + if name == d.f.target && d.f.rdGet != nil { + return apiv1.ResourceDefinition{}, d.f.rdGet + } + + if name == d.f.target && d.f.rdGetPasses != nil && d.f.rdGetPasses.Add(-1) == -1 { + return apiv1.ResourceDefinition{}, errStoreFault + } + + return d.ResourceDefinitionStore.Get(ctx, name) //nolint:wrapcheck // pass-through test double +} + +func (d faultyDefinitions) GetUncached(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + if name == d.f.target && d.f.rdGetUncached != nil && + (d.f.rdGetUncachedFails == nil || d.f.rdGetUncachedFails.Add(-1) >= 0) { + return apiv1.ResourceDefinition{}, d.f.rdGetUncached + } + + if name == d.f.target && d.f.rdGetUncachedPasses != nil && d.f.rdGetUncachedPasses.Add(-1) == -1 { + return apiv1.ResourceDefinition{}, errStoreFault + } + + return d.ResourceDefinitionStore.GetUncached(ctx, name) //nolint:wrapcheck // pass-through test double +} + +type faultyReplicas struct { + store.ResourceStore + + f storeFaults +} + +func (r faultyReplicas) Create(ctx context.Context, res *apiv1.Resource) error { + if res.Name == r.f.target && r.f.replicaCollide { + return store.ErrAlreadyExists + } + + return r.ResourceStore.Create(ctx, res) //nolint:wrapcheck // pass-through test double +} + +func (r faultyReplicas) PatchResourceSpec( + ctx context.Context, rdName, node string, mutate func(*apiv1.Resource) error, +) error { + if rdName == r.f.target && r.f.replicaPatch != nil { + return r.f.replicaPatch + } + + return r.ResourceStore.PatchResourceSpec(ctx, rdName, node, mutate) //nolint:wrapcheck // pass-through test double +} + +func (r faultyReplicas) Get(ctx context.Context, rdName, node string) (apiv1.Resource, error) { + if rdName == r.f.target && r.f.replicaGet != nil { + return apiv1.Resource{}, r.f.replicaGet + } + + return r.ResourceStore.Get(ctx, rdName, node) //nolint:wrapcheck // pass-through test double +} + +type faultyGroups struct { + store.ResourceGroupStore + + f storeFaults +} + +func (g faultyGroups) Get(ctx context.Context, name string) (apiv1.ResourceGroup, error) { + if g.f.groupGet != nil { + return apiv1.ResourceGroup{}, g.f.groupGet + } + + return g.ResourceGroupStore.Get(ctx, name) //nolint:wrapcheck // pass-through test double +} + +func (s faultyStore) Snapshots() store.SnapshotStore { + return faultySnapshots{s.Store.Snapshots(), s.f} +} + +func (s faultyStore) VolumeDefinitions() store.VolumeDefinitionStore { + return faultyVolumes{s.Store.VolumeDefinitions(), s.f} +} + +func (s faultyStore) ResourceDefinitions() store.ResourceDefinitionStore { + return faultyDefinitions{s.Store.ResourceDefinitions(), s.f} +} + +func (s faultyStore) Resources() store.ResourceStore { return faultyReplicas{s.Store.Resources(), s.f} } + +func (s faultyStore) ResourceGroups() store.ResourceGroupStore { + return faultyGroups{s.Store.ResourceGroups(), s.f} +} + +func seedMarkedCloneLeftover(t *testing.T, st store.Store, src, dst string) { + t.Helper() + + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{ + Name: dst, + Props: map[string]string{restoreFromSnapshotKey: restoreMarker(src, cloneSnapshotName(dst))}, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } +} + +// A clone resume that cannot read whether its internal snapshot is there does +// not go on as if it were, nor as if it were gone: it answers that the read +// failed and writes nothing. +func TestRDCloneResumeStopsWhenItCannotReadItsSnapshot(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-snapread") + seedMarkedCloneLeftover(t, backend, "src-snapread", "dst-snapread") + + base, stop := startServerWithStore(t, faultyStore{backend, storeFaults{target: "dst-snapread", snapshotList: errStoreFault}}) + defer stop() + + resp := postClone(t, base, "src-snapread", map[string]any{"name": "dst-snapread"}) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusInternalServerError { + t.Errorf("clone resume over an unreadable snapshot list = %d, want 500", resp.StatusCode) + } + + if vds, _ := backend.VolumeDefinitions().List(t.Context(), "dst-snapread"); len(vds) != 0 { + t.Errorf("the resume hydrated %d volume(s) without knowing its snapshot", len(vds)) + } +} + +// The judgement of a clone leftover that cannot read its volumes is no +// judgement: it answers the failure instead of calling the leftover anything. +func TestRDCloneStopsWhenItCannotReadTheLeftoversVolumes(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-vdread") + seedMarkedCloneLeftover(t, backend, "src-vdread", "dst-vdread") + + base, stop := startServerWithStore(t, faultyStore{backend, storeFaults{target: "dst-vdread", volumeList: errStoreFault}}) + defer stop() + + resp := postClone(t, base, "src-vdread", map[string]any{"name": "dst-vdread"}) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusInternalServerError { + t.Errorf("clone over a leftover whose volumes cannot be read = %d, want 500", resp.StatusCode) + } + + if snaps, _ := backend.Snapshots().ListByDefinition(t.Context(), "src-vdread"); len(snaps) != 0 { + t.Errorf("the clone took %d snapshot(s) without judging its leftover", len(snaps)) + } +} + +// A restore whose create collided cannot tell whose definition it met without +// reading it, and answers the read failure, not the collision. +func TestSnapshotRestoreAnswersAFailedReadOfTheDefinitionItCollidedWith(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-rdread") + seedRestoreSnapshot(t, backend, "src-rdread", "snap-rdread", []string{"node-a"}) + + base, stop := startServerWithStore(t, faultyStore{backend, storeFaults{ + target: "dst-rdread", rdCollides: true, rdGetUncached: errStoreFault, + }}) + defer stop() + + code, rc := restoreAnswer(t, base, "src-rdread", "snap-rdread", map[string]any{"to_resource": "dst-rdread"}) + if code != http.StatusInternalServerError { + t.Errorf("restore whose collided definition cannot be read = %d %q, want 500", code, rc.Message) + } +} + +// A replica create that collided with one that cannot be read back is not a +// replica placed. +func TestSnapshotRestoreDoesNotCountAReplicaItCannotReadBack(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-repread") + seedRestoreSnapshot(t, backend, "src-repread", "snap-repread", []string{"node-a"}) + + base, stop := startServerWithStore(t, faultyStore{backend, storeFaults{ + target: "dst-repread", replicaCollide: true, replicaGet: errStoreFault, + }}) + defer stop() + + code, rc := restoreAnswer(t, base, "src-repread", "snap-repread", + map[string]any{"to_resource": "dst-repread", "node_names": []string{"node-a"}}) + if code == http.StatusCreated { + t.Errorf("restore over a replica it could not read back = 201 %q", rc.Message) + } +} + +// A volume create that collided with one that cannot be read back is not a +// volume restored. +func TestSnapshotRestoreDoesNotCountAVolumeItCannotReadBack(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-volread") + seedRestoreSnapshot(t, backend, "src-volread", "snap-volread", []string{"node-a"}) + + base, stop := startServerWithStore(t, faultyStore{backend, storeFaults{ + target: "dst-volread", volumeCollides: true, volumeGet: errStoreFault, + }}) + defer stop() + + code, rc := restoreAnswer(t, base, "src-volread", "snap-volread", map[string]any{"to_resource": "dst-volread"}) + if code == http.StatusCreated { + t.Errorf("restore over a volume it could not read back = 201 %q", rc.Message) + } +} + +// A clone naming a group that cannot be read is told so, not told the group +// does not exist. +func TestRDCloneAnswersAnUnreadableGroupAsAReadFailure(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-rgread") + seedCloneRG(t, backend, "grp-rgread") + + base, stop := startServerWithStore(t, faultyStore{backend, storeFaults{groupGet: errStoreFault}}) + defer stop() + + resp := postClone(t, base, "src-rgread", map[string]any{"name": "dst-rgread", "resource_group": "grp-rgread"}) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusInternalServerError { + t.Errorf("clone into a group that cannot be read = %d, want 500", resp.StatusCode) + } +} + +// The creator's rollback that finds the definition adopted says it left it to +// the adopter, rather than reporting a rollback that never ran. +func TestTheCreatorsRollbackYieldsToAnAdopter(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-yield", Props: map[string]string{store.RestoreAdoptedProp: "2026-10-07T00:00:00Z"}, + }); err != nil { + t.Fatalf("seed the adopted definition: %v", err) + } + + s := &Server{Store: backend} + + if err := s.rollBackCompensating(ctx, "dst-yield", nil, rollbackEvenIfFinished); !errors.Is(err, errRollbackYielded) { + t.Errorf("rollback over an adopted definition = %v, want it to yield", err) + } + + if _, err := backend.ResourceDefinitions().Get(ctx, "dst-yield"); err != nil { + t.Errorf("the adopted definition was rolled back: %v", err) + } +} + +// The clone status a poll reads cannot call a leftover anything without its +// volumes: an unreadable listing is a 500 linstor-csi retries, not the 404 that +// sends it back to the POST as if the clone were unfinished. +func TestRDCloneStatusAnswersAnUnreadableLeftoverAsAReadFailure(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-stread") + seedMarkedCloneLeftover(t, backend, "src-stread", "dst-stread") + + base, stop := startServerWithStore(t, faultyStore{backend, storeFaults{target: "dst-stread", volumeList: errStoreFault}}) + defer stop() + + resp := httpGet(t, base+"/v1/resource-definitions/src-stread/clone/dst-stread") + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusInternalServerError { + t.Errorf("clone status over a leftover whose volumes cannot be read = %d, want 500", resp.StatusCode) + } +} + +// A finished restore's replay answers for a definition nobody verified, so a +// parent group it cannot read refuses it, as the clone's replay and the CLI +// restore do. It used to answer 201 with a warning, which binds a PV to a +// definition whose group may be gone. +func TestRestoreReplayRefusesAGroupItCannotRead(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, backend, "src-grpread") + seedRestoreSnapshot(t, backend, "src-grpread", "snap-grpread", []string{"node-a"}) + + if err := backend.ResourceGroups().Create(ctx, &apiv1.ResourceGroup{Name: "grp-read"}); err != nil { + t.Fatalf("seed the group: %v", err) + } + + plain, stopPlain := startServerWithStore(t, backend) + + body := map[string]any{"to_resource": "dst-grpread"} + if code, rc := restoreAnswer(t, plain, "src-grpread", "snap-grpread", body); code != http.StatusCreated { + stopPlain() + t.Fatalf("first restore = %d %q, want 201", code, rc.Message) + } + + stopPlain() + + if err := backend.ResourceDefinitions().PatchResourceDefinitionSpec(ctx, "dst-grpread", + func(rd *apiv1.ResourceDefinition) error { + rd.ResourceGroupName = "grp-read" + + return nil + }); err != nil { + t.Fatalf("parent the restored definition: %v", err) + } + + base, stop := startServerWithStore(t, faultyStore{backend, storeFaults{target: "dst-grpread", groupGet: errStoreFault}}) + defer stop() + + if code, rc := restoreAnswer(t, base, "src-grpread", "snap-grpread", body); code == http.StatusCreated { + t.Errorf("replay over a group that could not be read = %d %q, want a refusal", code, rc.Message) + } +} + +// A finished clone whose parent group cannot be read is not answered COMPLETE, +// and not sent back to the POST with a 404 either: a read failure is a 500, +// which linstor-csi returns from CreateVolume and retries. +func TestRDCloneStatusAnswersAnUnreadableGroupAsAReadFailure(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, backend, "src-stgrp") + + plain, stopPlain := startServerWithStore(t, backend) + + if code := cloneOnce(t, plain, "src-stgrp", "dst-stgrp", nil); code != http.StatusCreated { + stopPlain() + t.Fatalf("first clone = %d, want 201", code) + } + + stopPlain() + + if err := backend.ResourceGroups().Create(ctx, &apiv1.ResourceGroup{Name: "grp-stgrp"}); err != nil { + t.Fatalf("seed the group: %v", err) + } + + if err := backend.ResourceDefinitions().PatchResourceDefinitionSpec(ctx, "dst-stgrp", + func(rd *apiv1.ResourceDefinition) error { + rd.ResourceGroupName = "grp-stgrp" + + return nil + }); err != nil { + t.Fatalf("parent the clone: %v", err) + } + + base, stop := startServerWithStore(t, faultyStore{backend, storeFaults{target: "dst-stgrp", groupGet: errStoreFault}}) + defer stop() + + if code, status := cloneStatusOnce(t, base+"/v1/resource-definitions/src-stgrp/clone/dst-stgrp"); code != http.StatusInternalServerError { + t.Errorf("poll of a finished clone whose group cannot be read = %d %q, want 500", code, status) + } +} + +// A target that cannot be read is not a target that is absent. Read as absent, +// the create that followed collided and was adopted past every judgement of a +// leftover, and a replay of a finished clone or restore re-placed a replica on +// the node the operator had emptied. It is a 500 the caller retries. +func TestAnUnreadableTargetIsRefusedNotReplaced(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + faults storeFaults + replay func(t *testing.T, base string) int + }{ + { + name: "clone", + faults: storeFaults{target: "dst-unread", rdGet: errStoreFault}, + replay: func(t *testing.T, base string) int { + t.Helper() + + return cloneOnce(t, base, "src-unread", "dst-unread", nil) + }, + }, + { + name: "restore", + faults: storeFaults{ + target: "dst-unread", rdGetUncached: errStoreFault, rdGetUncachedFails: budgetOf(2), + }, + replay: func(t *testing.T, base string) int { + t.Helper() + + code, _ := restoreAnswer(t, base, "src-unread", "snap-unread", map[string]any{ + "to_resource": "dst-unread", "nodes": []string{"node-a", "node-b"}, + }) + + return code + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedTwoNodeSource(t, backend, "src-unread") + seedRestoreSnapshot(t, backend, "src-unread", "snap-unread", []string{"node-a", "node-b"}) + + plain, stopPlain := startServerWithStore(t, backend) + + if code := tc.replay(t, plain); code != http.StatusCreated { + stopPlain() + t.Fatalf("first %s = %d, want 201", tc.name, code) + } + + stopPlain() + + if err := backend.Resources().Delete(ctx, "dst-unread", "node-b"); err != nil { + t.Fatalf("empty node-b: %v", err) + } + + base, stop := startServerWithStore(t, faultyStore{backend, tc.faults}) + defer stop() + + if code := tc.replay(t, base); code != http.StatusInternalServerError { + t.Errorf("replay over a target that cannot be read = %d, want 500", code) + } + + replicas, err := backend.Resources().ListByDefinition(ctx, "dst-unread") + if err != nil || len(replicas) != 1 { + t.Errorf("the replay left replicas %v (%v), want node-a only", nodeNamesOf(replicas), err) + } + }) + } +} + +func budgetOf(n int32) *atomic.Int32 { + var b atomic.Int32 + + b.Store(n) + + return &b +} + +// Each read site that judges the target refuses on its own when it alone +// cannot read it: the four of them back each other up when every read fails, +// so a single one going back to "unreadable is absent" went unnoticed. A +// replay of a restore whose later read fails re-placed the emptied node, and a +// clone resume over an unfinished leftover was adopted past every judgement. +func TestAnUnreadableTargetIsRefusedAtEachReadSite(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + faults storeFaults + emptyA bool + replay func(t *testing.T, base string) int + wantAll int + }{ + { + name: "clone-target-state", + faults: storeFaults{target: "dst-site", rdGetPasses: budgetOf(2)}, + emptyA: true, + replay: func(t *testing.T, base string) int { + t.Helper() + + return cloneOnce(t, base, "src-site", "dst-site", nil) + }, + }, + { + name: "clone-replay", + faults: storeFaults{target: "dst-site", rdGetPasses: budgetOf(1)}, + wantAll: 1, + replay: func(t *testing.T, base string) int { + t.Helper() + + return cloneOnce(t, base, "src-site", "dst-site", nil) + }, + }, + { + name: "restore-prepared-target", + faults: storeFaults{target: "dst-site", rdGetUncachedPasses: budgetOf(0)}, + wantAll: 1, + replay: func(t *testing.T, base string) int { + t.Helper() + + code, _ := restoreAnswer(t, base, "src-site", "snap-site", map[string]any{ + "to_resource": "dst-site", "nodes": []string{"node-a", "node-b"}, + }) + + return code + }, + }, + { + name: "restore-target-state", + faults: storeFaults{target: "dst-site", rdGetUncachedPasses: budgetOf(2)}, + wantAll: 1, + replay: func(t *testing.T, base string) int { + t.Helper() + + code, _ := restoreAnswer(t, base, "src-site", "snap-site", map[string]any{ + "to_resource": "dst-site", "nodes": []string{"node-a", "node-b"}, + }) + + return code + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedTwoNodeSource(t, backend, "src-site") + seedRestoreSnapshot(t, backend, "src-site", "snap-site", []string{"node-a", "node-b"}) + + plain, stopPlain := startServerWithStore(t, backend) + + if code := tc.replay(t, plain); code != http.StatusCreated { + stopPlain() + t.Fatalf("first %s = %d, want 201", tc.name, code) + } + + stopPlain() + + if err := backend.Resources().Delete(ctx, "dst-site", "node-b"); err != nil { + t.Fatalf("empty node-b: %v", err) + } + + if tc.emptyA { + if err := backend.Resources().Delete(ctx, "dst-site", "node-a"); err != nil { + t.Fatalf("leave the clone unfinished: %v", err) + } + } + + base, stop := startServerWithStore(t, faultyStore{backend, tc.faults}) + defer stop() + + if code := tc.replay(t, base); code != http.StatusInternalServerError { + t.Errorf("replay whose one read of the target failed = %d, want 500", code) + } + + replicas, err := backend.Resources().ListByDefinition(ctx, "dst-site") + if err != nil || len(replicas) != tc.wantAll { + t.Errorf("the replay left replicas %v (%v), want %d", nodeNamesOf(replicas), err, tc.wantAll) + } + }) + } +} + +// The read that finds the stored name is the one that decides the name every +// later gate judges under. One that failed and was taken as "keep the +// request's spelling" let a blip that cleared by the next read judge a +// finished clone or restore under another case: no replica counted, and a +// replica re-placed on the node the operator had emptied. It is refused. +func TestAStoredNameThatCannotBeReadIsRefused(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + replay func(t *testing.T, base string) int + }{ + {name: "clone", replay: func(t *testing.T, base string) int { + t.Helper() + + return cloneOnce(t, base, "src-blip", "DST-BLIP", nil) + }}, + {name: "restore", replay: func(t *testing.T, base string) int { + t.Helper() + + code, _ := restoreAnswer(t, base, "src-blip", "snap-blip", map[string]any{ + "to_resource": "DST-BLIP", "nodes": []string{"node-a", "node-b"}, + }) + + return code + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedTwoNodeSource(t, backend, "src-blip") + seedRestoreSnapshot(t, backend, "src-blip", "snap-blip", []string{"node-a", "node-b"}) + + plain, stopPlain := startServerWithStore(t, backend) + + first := func() int { + if tc.name == "clone" { + return cloneOnce(t, plain, "src-blip", "dst-blip", nil) + } + + code, _ := restoreAnswer(t, plain, "src-blip", "snap-blip", map[string]any{ + "to_resource": "dst-blip", "nodes": []string{"node-a", "node-b"}, + }) + + return code + } + + if code := first(); code != http.StatusCreated { + stopPlain() + t.Fatalf("first %s = %d, want 201", tc.name, code) + } + + stopPlain() + + if err := backend.Resources().Delete(ctx, "dst-blip", "node-b"); err != nil { + t.Fatalf("empty node-b: %v", err) + } + + blip := faultyStore{caseFoldingDefinitions{backend}, storeFaults{target: "DST-BLIP", rdGetPasses: budgetOf(0)}} + + base, stop := startServerWithStore(t, blip) + defer stop() + + if code := tc.replay(t, base); code != http.StatusInternalServerError { + t.Errorf("replay whose stored-name read failed = %d, want 500", code) + } + + all, err := backend.Resources().List(ctx) + if err != nil { + t.Fatalf("list replicas: %v", err) + } + + var placed []string + + for i := range all { + if strings.EqualFold(all[i].Name, "dst-blip") { + placed = append(placed, all[i].Name+"@"+all[i].NodeName) + } + } + + if len(placed) != 1 { + t.Errorf("the replay left replicas %v, want only dst-blip@node-a", placed) + } + }) + } +} diff --git a/pkg/rest/vg_modify_golinstor_test.go b/pkg/rest/vg_modify_golinstor_test.go new file mode 100644 index 00000000..7205f6ac --- /dev/null +++ b/pkg/rest/vg_modify_golinstor_test.go @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/LINBIT/golinstor/client" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// The volume-group modify door takes golinstor's VolumeGroupModify whole: +// delete_namespaces drops every key below a namespace and leaves a key spelled +// as it, as upstream does. +func TestVolumeGroupModifyTakesGolinstorsWholeBody(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + + if err := st.ResourceGroups().Create(ctx, &apiv1.ResourceGroup{ + Name: "rg-vg", + VolumeGroups: []apiv1.VolumeGroup{{ + VolumeNumber: 0, + Props: map[string]string{"Aux": "x", "Aux/a": "y", "keep": "z"}, + }}, + }); err != nil { + t.Fatalf("seed the group: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(client.VolumeGroupModify{ + DeleteNamespaces: []string{"Aux"}, + }) + + resp := httpPut(t, base+"/v1/resource-groups/rg-vg/volume-groups/0", body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("vg modify with golinstor's body = %d, want 200", resp.StatusCode) + } + + rg, err := st.ResourceGroups().Get(ctx, "rg-vg") + if err != nil { + t.Fatalf("read the group: %v", err) + } + + vg := rg.VolumeGroups[0] + + if vg.Props["Aux"] != "x" { + t.Errorf("the key spelled as the namespace went: %v", vg.Props) + } + + if _, ok := vg.Props["Aux/a"]; ok { + t.Errorf("a key below the namespace stayed: %v", vg.Props) + } + + if vg.Props["keep"] != "z" { + t.Errorf("a key outside the namespace went: %v", vg.Props) + } +} + +// Flags are refused, and the refusal writes nothing: nothing sizes a spawned +// volume gross, so a 200 would answer for a change that does not happen. +func TestVolumeGroupModifyRefusesFlags(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + + if err := st.ResourceGroups().Create(ctx, &apiv1.ResourceGroup{ + Name: "rg-vgflags", + VolumeGroups: []apiv1.VolumeGroup{{VolumeNumber: 0, Props: map[string]string{"Aux/a": "y"}}}, + }); err != nil { + t.Fatalf("seed the group: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + body, _ := json.Marshal(client.VolumeGroupModify{ + DeleteNamespaces: []string{"Aux"}, + Flags: []string{"GROSS_SIZE"}, + }) + + resp := httpPut(t, base+"/v1/resource-groups/rg-vgflags/volume-groups/0", body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusNotImplemented { + t.Errorf("vg modify with flags = %d, want 501", resp.StatusCode) + } + + rg, err := st.ResourceGroups().Get(ctx, "rg-vgflags") + if err != nil { + t.Fatalf("read the group: %v", err) + } + + vg := rg.VolumeGroups[0] + + if len(vg.Flags) != 0 { + t.Errorf("a refused modify stored flags %v", vg.Flags) + } + + if vg.Props["Aux/a"] != "y" { + t.Errorf("a refused modify applied its namespace delete: %v", vg.Props) + } +} diff --git a/pkg/rest/volumes_per_resource.go b/pkg/rest/volumes_per_resource.go index aaa43ac0..6c491327 100644 --- a/pkg/rest/volumes_per_resource.go +++ b/pkg/rest/volumes_per_resource.go @@ -21,7 +21,6 @@ package rest import ( "fmt" "net/http" - "strings" apiv1 "github.com/cozystack/blockstor/pkg/api/v1" ) @@ -224,11 +223,5 @@ func mergeVolumePropsPatch(vol *apiv1.Volume, patch *apiv1.GenericPropsModify) { // I1: empty override value deletes the key (set-property KEY ""). applyPropsModify(vol.Props, patch.OverrideProps, patch.DeleteProps) - for _, ns := range patch.DeleteNamespace { - for k := range vol.Props { - if k == ns || strings.HasPrefix(k, ns+"/") { - delete(vol.Props, k) - } - } - } + deletePropNamespaces(vol.Props, patch.DeleteNamespace) } diff --git a/pkg/store/adoption_set_test.go b/pkg/store/adoption_set_test.go new file mode 100644 index 00000000..93d58b55 --- /dev/null +++ b/pkg/store/adoption_set_test.go @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store_test + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// claimPatchHook runs before on every spec patch, then lets it through, and +// fails it after it landed when landThenFail is set. +type claimPatchHook struct { + store.ResourceDefinitionStore + + before func() + landThenFail bool + failNoLand error + patches *atomic.Int32 + // cancel ends the caller's context with a landed write that fails, the + // way a write at the request's deadline does. + cancel context.CancelFunc +} + +// GetUncached refuses a dead context, as a real client does. +func (r claimPatchHook) GetUncached(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + if err := ctx.Err(); err != nil { + return apiv1.ResourceDefinition{}, err //nolint:wrapcheck // test double + } + + return r.ResourceDefinitionStore.GetUncached(ctx, name) //nolint:wrapcheck // test double +} + +func (r claimPatchHook) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + r.patches.Add(1) + + if r.before != nil { + r.before() + } + + if r.failNoLand != nil { + return r.failNoLand + } + + err := r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) + if err == nil && r.landThenFail { + if r.cancel != nil { + r.cancel() + } + + return context.DeadlineExceeded + } + + return err //nolint:wrapcheck // test double +} + +type claimPatchHookStore struct { + store.Store + + hook claimPatchHook +} + +func (s claimPatchHookStore) ResourceDefinitions() store.ResourceDefinitionStore { + h := s.hook + h.ResourceDefinitionStore = s.Store.ResourceDefinitions() + + if h.patches == nil { + h.patches = new(atomic.Int32) + } + + return h +} + +func seedClaimTarget(t *testing.T, st store.Store, name string, props map[string]string) { + t.Helper() + + if err := st.ResourceDefinitions().Create(t.Context(), + &apiv1.ResourceDefinition{Name: name, Props: props}); err != nil { + t.Fatalf("seed: %v", err) + } +} + +func claimTargetProps(t *testing.T, st store.Store, name string) map[string]string { + t.Helper() + + rd, err := st.ResourceDefinitions().Get(t.Context(), name) + if err != nil { + t.Fatalf("read back: %v", err) + } + + return rd.Props +} + +func claimSnap() *apiv1.Snapshot { + return &apiv1.Snapshot{VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 1024}}} +} + +// A claim over a definition the creator is rolling back writes nothing: the +// refusal is decided inside the one patch, before anything is written, so +// there is no mark and no volume record to take back off. One rolling back +// when the patch runs is refused the same as one already marked. +func TestAClaimOverARollbackWritesNothing(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + seeded string + late bool + }{ + {name: "rollback in progress", seeded: store.RollbackInProgress}, + {name: "rollback taking it apart", seeded: store.RollbackStepReapReplicas}, + {name: "rollback lands just before the claim", late: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + + props := map[string]string{store.RestoreVolumesProp: "0=2048"} + if tc.seeded != "" { + props[store.RollbackAbandonedProp] = tc.seeded + } + + seedClaimTarget(t, backend, "pvc-rb", props) + + var st store.Store = backend + if tc.late { + st = claimPatchHookStore{Store: backend, hook: claimPatchHook{before: func() { + _ = backend.ResourceDefinitions().PatchResourceDefinitionSpec(context.Background(), "pvc-rb", + func(rd *apiv1.ResourceDefinition) error { + rd.Props[store.RollbackAbandonedProp] = store.RollbackInProgress + + return nil + }) + }}} + } + + err := store.ClaimAdoptedLeftover(t.Context(), st, "pvc-rb", claimSnap()) + if !errors.Is(err, store.ErrAdoptedLeftoverRollingBack) { + t.Fatalf("claim over a rollback = %v, want ErrAdoptedLeftoverRollingBack", err) + } + + got := claimTargetProps(t, backend, "pvc-rb") + if mark, ok := got[store.RestoreAdoptedProp]; ok { + t.Errorf("a refused claim wrote adoption mark %q", mark) + } + + if got[store.RestoreVolumesProp] != "0=2048" { + t.Errorf("a refused claim changed the volume record to %q", got[store.RestoreVolumesProp]) + } + }) + } +} + +// An accepted claim writes the mark and the volumes snap holds; without a +// snapshot it records nothing. +func TestAnAcceptedClaimMarksAndRecords(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedClaimTarget(t, backend, "pvc-snap", nil) + seedClaimTarget(t, backend, "pvc-nosnap", map[string]string{store.RestoreVolumesProp: "0=2048"}) + + if err := store.ClaimAdoptedLeftover(t.Context(), backend, "pvc-snap", claimSnap()); err != nil { + t.Fatalf("claim: %v", err) + } + + if err := store.ClaimAdoptedLeftover(t.Context(), backend, "pvc-nosnap", nil); err != nil { + t.Fatalf("claim without a snapshot: %v", err) + } + + withSnap := claimTargetProps(t, backend, "pvc-snap") + if withSnap[store.RestoreAdoptedProp] == "" { + t.Error("an accepted claim left no adoption mark") + } + + if withSnap[store.RestoreVolumesProp] != "0=1024" { + t.Errorf("volume record = %q, want the claim's snapshot", withSnap[store.RestoreVolumesProp]) + } + + if got := claimTargetProps(t, backend, "pvc-nosnap")[store.RestoreVolumesProp]; got != "0=2048" { + t.Errorf("a claim without a snapshot changed the volume record to %q", got) + } +} + +// Claims that keep coming for one definition all answer, and the mark is one +// token, the last claim's. +func TestRepeatedClaimsAllAnswer(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedClaimTarget(t, backend, "pvc-many", nil) + + var last string + + for range 3 { + if err := store.ClaimAdoptedLeftover(t.Context(), backend, "pvc-many", claimSnap()); err != nil { + t.Fatalf("claim: %v", err) + } + + mark := claimTargetProps(t, backend, "pvc-many")[store.RestoreAdoptedProp] + if mark == "" || mark == last { + t.Fatalf("mark after a claim = %q, want a fresh token (previous %q)", mark, last) + } + + last = mark + } +} + +// A claim whose write landed and still answered an error is accepted: its own +// token is there to read back. +func TestAClaimWhoseWriteLandsAndErrsIsAccepted(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedClaimTarget(t, backend, "pvc-landed", nil) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + st := claimPatchHookStore{Store: backend, hook: claimPatchHook{landThenFail: true, cancel: cancel}} + + if err := store.ClaimAdoptedLeftover(ctx, st, "pvc-landed", claimSnap()); err != nil { + t.Errorf("a claim whose write landed = %v, want it accepted", err) + } +} + +var errRefused = errors.New("apiserver refused the patch") + +// A claim whose write did not land answers the error and leaves no mark. +func TestAClaimWhoseWriteFailsAnswersTheError(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedClaimTarget(t, backend, "pvc-failed", nil) + + st := claimPatchHookStore{Store: backend, hook: claimPatchHook{failNoLand: errRefused}} + + if err := store.ClaimAdoptedLeftover(t.Context(), st, "pvc-failed", claimSnap()); !errors.Is(err, errRefused) { + t.Errorf("a claim whose write failed = %v, want the write's error", err) + } + + if mark, ok := claimTargetProps(t, backend, "pvc-failed")[store.RestoreAdoptedProp]; ok { + t.Errorf("a failed claim left adoption mark %q", mark) + } +} + +// A definition recreated under the name before the claim's patch is marked +// like any other: an adoption mark only ever makes a rollback leave a +// definition in place. +func TestAClaimMarksADefinitionRecreatedUnderTheName(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedClaimTarget(t, backend, "pvc-recreated", nil) + + st := claimPatchHookStore{Store: backend, hook: claimPatchHook{before: func() { + _ = backend.ResourceDefinitions().Delete(context.Background(), "pvc-recreated") + _ = backend.ResourceDefinitions().Create(context.Background(), &apiv1.ResourceDefinition{Name: "pvc-recreated"}) + }}} + + if err := store.ClaimAdoptedLeftover(t.Context(), st, "pvc-recreated", nil); err != nil { + t.Fatalf("claim: %v", err) + } + + if claimTargetProps(t, backend, "pvc-recreated")[store.RestoreAdoptedProp] == "" { + t.Error("the recreated definition carries no adoption mark") + } +} + +// The CLI door's refusal over a rollback still in progress does not warn of a +// clear handing the definition to a retry the rollback then deletes: the +// rollback checks its own mark before its first delete, so a clear stops it. +func TestTheInProgressRefusalSaysAClearStopsTheRollback(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-inprog", + Props: map[string]string{ + store.RestoreFromSnapshotProp: "pvc-src:snap-1", + store.RollbackAbandonedProp: store.RollbackInProgress, + }, + }); err != nil { + t.Fatalf("seed: %v", err) + } + + snap := &apiv1.Snapshot{ + ResourceName: "pvc-src", Name: "snap-1", + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 1024}}, + } + + _, _, err := store.JudgeRestoreLeftover(ctx, backend, "pvc-inprog", snap) + if err == nil { + t.Fatal("a leftover under a rollback in progress was not refused") + } + + if !strings.Contains(err.Error(), "stops that rollback before it deletes anything") || + strings.Contains(err.Error(), "rollback then deletes") { + t.Errorf("refusal = %q, want it to say a clear stops the rollback", err) + } +} diff --git a/pkg/store/k8s/prepared_target_uncached_test.go b/pkg/store/k8s/prepared_target_uncached_test.go new file mode 100644 index 00000000..3425a4fd --- /dev/null +++ b/pkg/store/k8s/prepared_target_uncached_test.go @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: Apache-2.0 + +package k8s_test + +import ( + "context" + "testing" + + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + crdv1alpha1 "github.com/cozystack/blockstor/api/v1alpha1" + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store/k8s" +) + +// cacheBehindTheTarget is a client whose cache has seen neither the target's +// volumes nor its replicas: the definition reads back without its inline +// volume definitions, and no replica is listed. +type cacheBehindTheTarget struct{ ctrlclient.Client } + +func (c cacheBehindTheTarget) Get( + ctx context.Context, key ctrlclient.ObjectKey, obj ctrlclient.Object, opts ...ctrlclient.GetOption, +) error { + err := c.Client.Get(ctx, key, obj, opts...) + if rd, ok := obj.(*crdv1alpha1.ResourceDefinition); ok && err == nil { + rd.Spec.VolumeDefinitions = nil + } + + return err //nolint:wrapcheck // pass-through +} + +func (c cacheBehindTheTarget) List(ctx context.Context, list ctrlclient.ObjectList, opts ...ctrlclient.ListOption) error { + if _, ok := list.(*crdv1alpha1.ResourceList); ok { + return nil + } + + return c.Client.List(ctx, list, opts...) //nolint:wrapcheck // pass-through +} + +// The prepared-target decision reads a definition's volumes and replicas past +// the cache: the volume restore and the resource restore after it can be +// served by different replicas of the server. +func TestVolumeAndReplicaListingsReadPastTheCache(t *testing.T) { + if fixture == nil { + t.Skip("envtest assets not installed; run `make setup-envtest` to enable") + } + + ctx := t.Context() + seed := k8s.New(fixture.client) + + if err := seed.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "pvc-prep-uncached"}); err != nil { + t.Fatalf("seed definition: %v", err) + } + + t.Cleanup(func() { _ = seed.ResourceDefinitions().Delete(context.Background(), "pvc-prep-uncached") }) + + if err := seed.VolumeDefinitions().Create(ctx, "pvc-prep-uncached", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 1 << 20}); err != nil { + t.Fatalf("seed volume: %v", err) + } + + if err := seed.Resources().Create(ctx, &apiv1.Resource{Name: "pvc-prep-uncached", NodeName: "node-p"}); err != nil { + t.Fatalf("seed replica: %v", err) + } + + t.Cleanup(func() { _ = seed.Resources().Delete(context.Background(), "pvc-prep-uncached", "node-p") }) + + st := k8s.NewWithAPIReader(cacheBehindTheTarget{fixture.client}, fixture.client) + + if vds, _ := st.VolumeDefinitions().List(ctx, "pvc-prep-uncached"); len(vds) != 0 { + t.Fatalf("fixture: the cached volume listing was supposed to be empty, got %d", len(vds)) + } + + vols, ok := st.VolumeDefinitions().(interface { + ListUncached(ctx context.Context, rdName string) ([]apiv1.VolumeDefinition, error) + }) + if !ok { + t.Fatal("the Kubernetes volume-definition store has no ListUncached") + } + + if vds, err := vols.ListUncached(ctx, "pvc-prep-uncached"); err != nil || len(vds) != 1 { + t.Errorf("ListUncached = %d volume(s) (%v), want the one the API server holds", len(vds), err) + } + + if replicas, _ := st.Resources().ListByDefinition(ctx, "pvc-prep-uncached"); len(replicas) != 0 { + t.Fatalf("fixture: the cached replica listing was supposed to be empty, got %d", len(replicas)) + } + + reps, ok := st.Resources().(interface { + ListByDefinitionUncached(ctx context.Context, rdName string) ([]apiv1.Resource, error) + }) + if !ok { + t.Fatal("the Kubernetes resource store has no ListByDefinitionUncached") + } + + if replicas, err := reps.ListByDefinitionUncached(ctx, "pvc-prep-uncached"); err != nil || len(replicas) != 1 { + t.Errorf("ListByDefinitionUncached = %d replica(s) (%v), want the one the API server holds", len(replicas), err) + } +} diff --git a/pkg/store/k8s/resource_get_uncached_test.go b/pkg/store/k8s/resource_get_uncached_test.go new file mode 100644 index 00000000..e1bf7a07 --- /dev/null +++ b/pkg/store/k8s/resource_get_uncached_test.go @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: Apache-2.0 + +package k8s_test + +import ( + "context" + "errors" + "testing" + + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + crdv1alpha1 "github.com/cozystack/blockstor/api/v1alpha1" + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" + "github.com/cozystack/blockstor/pkg/store/k8s" +) + +// replicaCacheIsBlind is a client whose cache has not seen any replica. +type replicaCacheIsBlind struct{ ctrlclient.Client } + +func (b replicaCacheIsBlind) Get( + ctx context.Context, key ctrlclient.ObjectKey, obj ctrlclient.Object, opts ...ctrlclient.GetOption, +) error { + if _, ok := obj.(*crdv1alpha1.Resource); ok { + return b.Client.Get(ctx, ctrlclient.ObjectKey{Name: "no-such-" + key.Name}, obj, opts...) //nolint:wrapcheck // pass-through + } + + return b.Client.Get(ctx, key, obj, opts...) //nolint:wrapcheck // pass-through +} + +// A Create that collided with a replica decides on whether that replica is +// still there and whether it is going, so the read behind it is the API +// server's, not a cache's. +func TestResourceGetUncachedReadsPastTheCache(t *testing.T) { + if fixture == nil { + t.Skip("envtest assets not installed; run `make setup-envtest` to enable") + } + + ctx := t.Context() + seed := k8s.New(fixture.client) + + if err := seed.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "pvc-res-uncached"}); err != nil { + t.Fatalf("seed definition: %v", err) + } + + t.Cleanup(func() { _ = seed.ResourceDefinitions().Delete(context.Background(), "pvc-res-uncached") }) + + if err := seed.Resources().Create(ctx, &apiv1.Resource{Name: "pvc-res-uncached", NodeName: "node-x"}); err != nil { + t.Fatalf("seed replica: %v", err) + } + + t.Cleanup(func() { _ = seed.Resources().Delete(context.Background(), "pvc-res-uncached", "node-x") }) + + st := k8s.NewWithAPIReader(replicaCacheIsBlind{fixture.client}, fixture.client) + + if _, err := st.Resources().Get(ctx, "pvc-res-uncached", "node-x"); err == nil { + t.Fatal("fixture: the cached read was supposed to miss the replica") + } + + direct, ok := st.Resources().(interface { + GetUncached(ctx context.Context, rdName, node string) (apiv1.Resource, error) + }) + if !ok { + t.Fatal("the Kubernetes resource store has no GetUncached") + } + + if _, err := direct.GetUncached(ctx, "pvc-res-uncached", "node-x"); err != nil { + t.Errorf("GetUncached missed a replica the API server holds: %v", err) + } +} + +var errLiveReadRefused = errors.New("apiserver refused the live read") + +// liveReadRefused is an API reader that cannot be reached. +type liveReadRefused struct{ ctrlclient.Reader } + +func (liveReadRefused) Get(context.Context, ctrlclient.ObjectKey, ctrlclient.Object, ...ctrlclient.GetOption) error { + return errLiveReadRefused +} + +// The live read is what a collision decides on, so a failed one is an error, +// never "not found" and never the cache's answer in its place. +func TestResourceGetUncachedReportsAFailedLiveRead(t *testing.T) { + t.Parallel() + + st := k8s.NewWithAPIReader(bug201NewFakeClient(t), liveReadRefused{}) + + direct, ok := st.Resources().(interface { + GetUncached(ctx context.Context, rdName, node string) (apiv1.Resource, error) + }) + if !ok { + t.Fatal("the Kubernetes resource store has no GetUncached") + } + + _, err := direct.GetUncached(t.Context(), "pvc-live-refused", "node-x") + if !errors.Is(err, errLiveReadRefused) || errors.Is(err, store.ErrNotFound) { + t.Errorf("GetUncached over a failed live read = %v, want that error and not NotFound", err) + } +} diff --git a/pkg/store/k8s/resources.go b/pkg/store/k8s/resources.go index 89ea4fa0..f1e933fc 100644 --- a/pkg/store/k8s/resources.go +++ b/pkg/store/k8s/resources.go @@ -123,17 +123,15 @@ func (s *resources) ListByDefinition(ctx context.Context, rdName string) ([]apiv // A selectable FIELD does not have that failure mode: it selects on the // spec value every replica carries, whoever wrote it, and a server that // cannot serve the selector says so instead of answering short. - out, err := s.listScoped(ctx, s.c, FieldResourceDefinitionName, rdName, - func(r *crdv1alpha1.Resource) bool { return r.Spec.ResourceDefinitionName == rdName }) - if err != nil { - return nil, err - } - - // Every replica here belongs to the same definition, so the node is what - // distinguishes them. - sort.Slice(out, func(i, j int) bool { return out[i].NodeName < out[j].NodeName }) + return s.listByDefinitionThrough(ctx, s.c, rdName) +} - return out, nil +// ListByDefinitionUncached answers what ListByDefinition answers from the API +// server when the store has a direct reader, for a decision that cannot take a +// cache's word: whether a definition another replica of this server just +// prepared holds a live replica. +func (s *resources) ListByDefinitionUncached(ctx context.Context, rdName string) ([]apiv1.Resource, error) { + return s.listByDefinitionThrough(ctx, s.nodeScopedReader(), rdName) } func (s *resources) Get(ctx context.Context, rdName, node string) (apiv1.Resource, error) { @@ -151,6 +149,28 @@ func (s *resources) Get(ctx context.Context, rdName, node string) (apiv1.Resourc return crdToWireResource(&crd), nil } +// GetUncached answers what Get answers from the API server when the store has +// a direct reader, for a decision that cannot take a cache's word: whether a +// replica a Create collided with is still there, and whether it is going. +func (s *resources) GetUncached(ctx context.Context, rdName, node string) (apiv1.Resource, error) { + if s.apiReader == nil { + return s.Get(ctx, rdName, node) + } + + var crd crdv1alpha1.Resource + + err := s.apiReader.Get(ctx, types.NamespacedName{Name: resourceCRDName(rdName, node)}, &crd) + if err != nil { + if apierrors.IsNotFound(err) { + return apiv1.Resource{}, errors.Wrapf(store.ErrNotFound, "resource %q on node %q", rdName, node) + } + + return apiv1.Resource{}, errors.Wrapf(err, "get Resource %s/%s live", rdName, node) + } + + return crdToWireResource(&crd), nil +} + func (s *resources) Create(ctx context.Context, in *apiv1.Resource) error { if in == nil { return errors.New("nil Resource") @@ -522,6 +542,22 @@ func (s *resources) ClearDRBDPort(ctx context.Context, rdName, node string) erro return nil } +func (s *resources) listByDefinitionThrough( + ctx context.Context, reader ctrlclient.Reader, rdName string, +) ([]apiv1.Resource, error) { + out, err := s.listScoped(ctx, reader, FieldResourceDefinitionName, rdName, + func(r *crdv1alpha1.Resource) bool { return r.Spec.ResourceDefinitionName == rdName }) + if err != nil { + return nil, err + } + + // Every replica here belongs to the same definition, so the node is what + // distinguishes them. + sort.Slice(out, func(i, j int) bool { return out[i].NodeName < out[j].NodeName }) + + return out, nil +} + // buildVolumeStatusForApply turns a slice of per-volume // observations into the SSA-shaped Status.Volumes payload. Only // non-empty fields land in the apply object so SSA doesn't claim diff --git a/pkg/store/k8s/volume_definitions.go b/pkg/store/k8s/volume_definitions.go index 8fed3eb6..1e17679d 100644 --- a/pkg/store/k8s/volume_definitions.go +++ b/pkg/store/k8s/volume_definitions.go @@ -60,6 +60,24 @@ func (s *volumeDefinitions) List(ctx context.Context, rdName string) ([]apiv1.Vo return nil, err } + return wireVDsOf(rd), nil +} + +// ListUncached answers what List answers from the API server when the store +// has a direct reader. A volume restore served by one replica of this server +// and the resource restore after it served by another is linstor-csi's normal +// sequence, and the second must not judge the first's volumes off a cache that +// has not seen them. +func (s *volumeDefinitions) ListUncached(ctx context.Context, rdName string) ([]apiv1.VolumeDefinition, error) { + rd, err := s.fetchRDLive(ctx, rdName) + if err != nil { + return nil, err + } + + return wireVDsOf(rd), nil +} + +func wireVDsOf(rd *crdv1alpha1.ResourceDefinition) []apiv1.VolumeDefinition { out := make([]apiv1.VolumeDefinition, 0, len(rd.Spec.VolumeDefinitions)) for i := range rd.Spec.VolumeDefinitions { out = append(out, crdToWireVD(&rd.Spec.VolumeDefinitions[i])) @@ -67,7 +85,7 @@ func (s *volumeDefinitions) List(ctx context.Context, rdName string) ([]apiv1.Vo sort.Slice(out, func(i, j int) bool { return out[i].VolumeNumber < out[j].VolumeNumber }) - return out, nil + return out } // ListAll reads every definition's inline volumes from one list of the diff --git a/pkg/store/leftover.go b/pkg/store/leftover.go new file mode 100644 index 00000000..a9cf9246 --- /dev/null +++ b/pkg/store/leftover.go @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" +) + +// LeftoverProgress is where a target carrying a clone's or a restore's marker +// stands. Every door that resumes one asks it: the REST clone and its status +// poll, the REST restore and the CLI restore. They have to get the same +// answer, or one door finishes what its twin refuses. +type LeftoverProgress int + +const ( + // LeftoverUnfinished is not done and not broken: no volumes yet, some of + // the snapshot's volumes still missing, or no replica holding them. The + // resume finishes it. + LeftoverUnfinished LeftoverProgress = iota + // LeftoverFinished holds every volume and, where one is needed, at least + // one replica. + LeftoverFinished + // LeftoverForeign carries a volume the restore would not have written. + LeftoverForeign + // LeftoverTearingDown still lists replicas, but every one of them is + // already accepted for deletion. Neither finished nor safe to resume: the + // deletion is reaping what completing it would write. + LeftoverTearingDown +) + +// AssessLeftover is the one judgement of a marker-bearing leftover. targetVDs +// may be empty. snap is the point-in-time the leftover was restored from, or +// nil once that is gone. A clone needs a replica to hold its data; a restore +// needs one only when the request placed replicas at all, since a bare restore +// leaves an empty shell by design. +// +// The replicas are read first, because a tear-down decides the answer whatever +// the volumes say. A leftover the resume would write into, one with no volumes +// or missing one the snapshot recorded, is refused while every replica under +// the name is stamped for deletion: hydrating into it races the tear-down +// reaping what it writes. A leftover that would be answered finished is refused +// the same way when it needs a replica to be finished, since none of its +// replicas holds the data any more. A finished bare restore needs none, so its +// replay stays a replay. +// +// Where the replicas are is not part of the question. One replica with a disk +// holds the data, and which nodes carry it afterwards is placement, which ordinary +// operations change: a resume that topped up every node the request named +// re-stamped a replica on a node the operator had emptied, restored from the +// point-in-time beside a replica that had moved on with live writes. +// +// A volume number the snapshot never recorded is judged by what else the target +// holds. Beside a missing snapshot volume it is somebody else's: the resume +// would hydrate around it and report complete a definition carrying a volume +// nothing restored. Where a replica is needed and there is none, the same. +// Beside every snapshot volume it is the target's own, added after the copy +// finished the way a volume is expanded: on a clone or a placed restore once it +// also has a replica, and on a bare restore, which places nothing, by those +// volumes alone. Refusing it would tell the operator to delete a working +// definition. +func AssessLeftover( + ctx context.Context, st Store, targetName string, + targetVDs []apiv1.VolumeDefinition, snap *apiv1.Snapshot, needReplica bool, +) (LeftoverProgress, error) { + count, err := CountReplicas(ctx, st, targetName) + if err != nil { + return LeftoverUnfinished, err + } + + tearingDown := count.Total > 0 && count.Live == 0 + + unfinished := LeftoverUnfinished + if tearingDown { + unfinished = LeftoverTearingDown + } + + if len(targetVDs) == 0 { + return unfinished, nil + } + + snap, err = referenceShape(ctx, st, targetName, snap) + if err != nil { + return LeftoverUnfinished, err + } + + var shape leftoverShape + + if snap != nil { + shape = leftoverAgainstSnapshot(snap, targetVDs) + + switch { + case shape.smaller, shape.missing && shape.extra: + return LeftoverForeign, nil + case shape.missing: + return unfinished, nil + } + } + + switch { + case !needReplica: + return LeftoverFinished, nil + case count.Holding > 0: + return LeftoverFinished, nil + case tearingDown: + return LeftoverTearingDown, nil + case shape.extra: + return LeftoverForeign, nil + default: + return LeftoverUnfinished, nil + } +} + +// referenceShape is what a leftover is judged against: the snapshot the caller +// hands in, and without one the volumes the definition recorded, which is what +// every rollback's judgement reads whether or not the snapshot still exists. +func referenceShape(ctx context.Context, st Store, targetName string, snap *apiv1.Snapshot) (*apiv1.Snapshot, error) { + if snap != nil { + return snap, nil + } + + return recordedSnapshotShape(ctx, st, targetName) +} + +// recordedSnapshotShape stands in for a snapshot that is gone with the volumes +// the definition recorded when it was restored from it (RestoreVolumesProp), +// so a leftover is still judged against what the snapshot held. nil means no +// record: the leftover is judged on its own volumes, as before the record +// existed. +func recordedSnapshotShape(ctx context.Context, st Store, targetName string) (*apiv1.Snapshot, error) { + rd, err := st.ResourceDefinitions().GetUncached(ctx, targetName) + if errors.Is(err, ErrNotFound) { + return nil, nil //nolint:nilnil // no definition, no record + } + + if err != nil { + return nil, fmt.Errorf("read %q for its recorded volumes: %w", targetName, err) + } + + vols, ok := RecordedRestoreVolumes(rd.Props) + if !ok { + return nil, nil //nolint:nilnil // no record: judged as before + } + + return &apiv1.Snapshot{VolumeDefinitions: vols}, nil +} + +// CountReplicas counts the replicas under a name: how many hold the data, how +// many are not stamped for deletion, and all of them. A replica stamped for +// deletion does not hold the data: its satellite finalizer may keep it listed +// for as long as the owning node is down, and a replay answered 201 over it +// binds a volume to a definition that is going away. Nor does a diskless or +// tie-breaker one, which keeps a definition from reading as torn down and +// never makes it finished: answered 201 over, it reports a volume restored +// with no copy of the data anywhere. +// +// The list is the API server's where the store has a direct reader: every +// caller decides from it whether a leftover is finished, torn down or still to +// be completed, and a lagging informer answers all three wrong, a replica +// whose delete it has not seen yet counted live, one just stamped not at all. +func CountReplicas(ctx context.Context, st Store, rdName string) (ReplicaCount, error) { + replicas, err := LiveReplicas(ctx, st, rdName) + if err != nil { + return ReplicaCount{}, fmt.Errorf("list the replicas of %q: %w", rdName, err) + } + + count := ReplicaCount{Total: len(replicas)} + + for i := range replicas { + if slices.Contains(replicas[i].Flags, apiv1.ResourceFlagDelete) { + continue + } + + count.Live++ + + if HoldsData(&replicas[i]) { + count.Holding++ + } + } + + return count, nil +} + +// ReplicaCount is what CountReplicas found under a definition. +type ReplicaCount struct { + // Holding is the replicas that hold the data: not stamped for deletion, + // and neither diskless nor a tie-breaker. + Holding int + // Live is the replicas not stamped for deletion. + Live int + // Total is every replica listed. + Total int +} + +// HoldsData reports whether a replica has a disk of its own. A diskless or +// tie-breaker replica reads its peers' data over the network and has none. +func HoldsData(res *apiv1.Resource) bool { + return !slices.Contains(res.Flags, apiv1.ResourceFlagDiskless) && + !slices.Contains(res.Flags, apiv1.ResourceFlagTieBreaker) +} + +// MissingReplicas names the nodes in wanted that hold no replica of rdName +// with a disk, read past the cache. A rollback that leaves a definition because it +// already reads as finished uses it to say which of the replicas the request +// asked for never landed: a finished definition is not re-placed by a retry, +// so the caller has to place them. +func MissingReplicas(ctx context.Context, st Store, rdName string, wanted []string) ([]string, error) { + replicas, err := LiveReplicas(ctx, st, rdName) + if err != nil { + return nil, fmt.Errorf("list the replicas of %q: %w", rdName, err) + } + + var missing []string + + for _, node := range wanted { + placed := slices.ContainsFunc(replicas, func(r apiv1.Resource) bool { + return strings.EqualFold(r.NodeName, node) && !slices.Contains(r.Flags, apiv1.ResourceFlagDelete) && + HoldsData(&r) + }) + if !placed { + missing = append(missing, node) + } + } + + return missing, nil +} + +// leftoverShape is how a target's volumes compare with the snapshot's. +type leftoverShape struct { + // missing: a volume the snapshot recorded is not on the target yet. A + // multi-volume restore whose first attempt died between two + // VolumeDefinitions().Create calls has this. + missing bool + // smaller: a volume is on the target at less than the snapshot captured. + smaller bool + // extra: the target holds a volume number the snapshot never recorded. + extra bool +} + +// leftoverAgainstSnapshot compares a leftover target's volumes with the +// snapshot it would be restored from. +// +// Comparing volume COUNTS here would refuse the partial: hydration tolerates a +// volume already present so it can be finished. +// +// Larger than captured is still the target's own: a finished clone may be +// expanded like any volume, and an ordinary ControllerExpandVolume on it must +// not turn every later replay into a refusal whose correction deletes a clone +// holding data. Hydration never writes a smaller volume and nothing shrinks +// one, so smaller is the shape that is somebody else's. +func leftoverAgainstSnapshot(snap *apiv1.Snapshot, targetVDs []apiv1.VolumeDefinition) leftoverShape { + var shape leftoverShape + + captured := make(map[int32]int64, len(snap.VolumeDefinitions)) + for _, vol := range snap.VolumeDefinitions { + captured[vol.VolumeNumber] = vol.SizeKib + } + + present := make(map[int32]struct{}, len(targetVDs)) + + for i := range targetVDs { + present[targetVDs[i].VolumeNumber] = struct{}{} + + was, ok := captured[targetVDs[i].VolumeNumber] + + switch { + case !ok: + shape.extra = true + case targetVDs[i].SizeKib < was: + shape.smaller = true + } + } + + for number := range captured { + if _, ok := present[number]; !ok { + shape.missing = true + } + } + + return shape +} diff --git a/pkg/store/leftover_test.go b/pkg/store/leftover_test.go new file mode 100644 index 00000000..ffd4bbcd --- /dev/null +++ b/pkg/store/leftover_test.go @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: Apache-2.0 + +package store_test + +import ( + "context" + "errors" + "strings" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// cacheMissesTheDelete serves a replica list from a cache that has not seen +// the DELETE stamp yet; the API server has. +type cacheMissesTheDelete struct{ store.ResourceStore } + +func (c cacheMissesTheDelete) ListByDefinition(ctx context.Context, rdName string) ([]apiv1.Resource, error) { + replicas, err := c.ResourceStore.ListByDefinition(ctx, rdName) + for i := range replicas { + replicas[i].Flags = nil + } + + return replicas, err //nolint:wrapcheck // pass-through test double +} + +func (c cacheMissesTheDelete) ListByDefinitionUncached(ctx context.Context, rdName string) ([]apiv1.Resource, error) { + return c.ResourceStore.ListByDefinition(ctx, rdName) //nolint:wrapcheck // pass-through test double +} + +type cacheMissesTheDeleteStore struct{ store.Store } + +func (c cacheMissesTheDeleteStore) Resources() store.ResourceStore { + return cacheMissesTheDelete{c.Store.Resources()} +} + +// Every leftover decision counts replicas, and a lagging informer that still +// showed a replica being deleted as live called a restore finished over a +// definition that is going away. +func TestCountReplicasReadsPastTheCache(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + if err := backend.Resources().Create(t.Context(), &apiv1.Resource{ + Name: "pvc-dst", NodeName: "node-a", Flags: []string{apiv1.ResourceFlagDelete}, + }); err != nil { + t.Fatalf("seed the replica being deleted: %v", err) + } + + count, err := store.CountReplicas(t.Context(), cacheMissesTheDeleteStore{backend}, "pvc-dst") + if err != nil { + t.Fatalf("count: %v", err) + } + + if count.Live != 0 || count.Total != 1 { + t.Errorf("live/total = %d/%d, want 0/1: the cache's view of the replica decided", count.Live, count.Total) + } +} + +// A rollback still running is not one that gave up, and the refusal says so +// rather than telling the operator to delete a definition that is about to go +// on its own. +func TestJudgeRestoreLeftoverWordsARollbackStillInProgress(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-dst", Props: map[string]string{ + store.RestoreFromSnapshotProp: "pvc-src:snap", + store.RollbackAbandonedProp: store.RollbackInProgress, + }, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + _, own, err := store.JudgeRestoreLeftover(ctx, backend, "pvc-dst", &apiv1.Snapshot{ + Name: "snap", ResourceName: "pvc-src", + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 1 << 20}}, + }) + if !own || !errors.Is(err, store.ErrRestoreRollbackAbandoned) { + t.Fatalf("judge = own %v, %v; want a refusal", own, err) + } + + if !strings.Contains(err.Error(), "being rolled back") { + t.Errorf("refusal %q does not say a rollback is still running", err) + } +} + +// A diskless or tie-breaker replica holds no copy of the data, so a leftover +// carrying every volume and only such a replica is not finished: answered as +// done, it reported a volume restored with no data on any node. Nor is it torn +// down, since that replica is not going away. +func TestAssessLeftoverDoesNotCountAReplicaWithoutDataAsFinished(t *testing.T) { + t.Parallel() + + for _, flag := range []string{apiv1.ResourceFlagDiskless, apiv1.ResourceFlagTieBreaker} { + t.Run(flag, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + if err := backend.Resources().Create(t.Context(), &apiv1.Resource{ + Name: "pvc-nodata", NodeName: "node-a", Flags: []string{flag}, + }); err != nil { + t.Fatalf("seed the replica without data: %v", err) + } + + vds := []apiv1.VolumeDefinition{{VolumeNumber: 0, SizeKib: 1024}} + snap := &apiv1.Snapshot{VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 1024}}} + + progress, err := store.AssessLeftover(t.Context(), backend, "pvc-nodata", vds, snap, true) + if err != nil { + t.Fatalf("assess: %v", err) + } + + if progress != store.LeftoverUnfinished { + t.Errorf("leftover whose only replica is %s = %v, want unfinished", flag, progress) + } + }) + } +} + +// A rollback that leaves a finished definition names the requested nodes that +// hold no copy of the data, and a diskless replica there is not one. +func TestMissingReplicasNamesANodeHoldingOnlyADisklessReplica(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + + for _, res := range []apiv1.Resource{ + {Name: "pvc-miss", NodeName: "node-a"}, + {Name: "pvc-miss", NodeName: "node-b", Flags: []string{apiv1.ResourceFlagDiskless}}, + {Name: "pvc-miss", NodeName: "node-c", Flags: []string{apiv1.ResourceFlagDelete}}, + } { + if err := backend.Resources().Create(t.Context(), &res); err != nil { + t.Fatalf("seed the replica on %s: %v", res.NodeName, err) + } + } + + missing, err := store.MissingReplicas(t.Context(), backend, "pvc-miss", []string{"node-a", "node-b", "node-c"}) + if err != nil { + t.Fatalf("missing replicas: %v", err) + } + + if strings.Join(missing, ",") != "node-b,node-c" { + t.Errorf("missing = %v, want [node-b node-c]", missing) + } +} diff --git a/pkg/store/local_props.go b/pkg/store/local_props.go index 6ed6b321..31065b60 100644 --- a/pkg/store/local_props.go +++ b/pkg/store/local_props.go @@ -18,16 +18,123 @@ limitations under the License. package store -import "maps" +import ( + "errors" + "maps" + "slices" + "strconv" + "strings" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" +) + +// RestoreFromSnapshotProp is the marker a restored or cloned definition +// carries, `:`. The satellite reads it to restore +// from the snapshot instead of creating a blank volume, and the resume paths +// read it to recognise their own leftover. +const RestoreFromSnapshotProp = "BlockstorRestoreFromSnapshot" // RollbackAbandonedProp marks a resource definition whose compensation started // and did not report finishing, with the step it stopped at. The replay gate of // the clone path refuses over it. const RollbackAbandonedProp = "BlockstorRollbackAbandoned" -// restoreFromSnapshotMarker is the `:` a restored or cloned -// definition carries; see TravellingProps for why it never travels. -const restoreFromSnapshotMarker = "BlockstorRestoreFromSnapshot" +// RollbackInProgress is the value RollbackAbandonedProp carries while a +// rollback is running, before it reports the step it stopped at. +const RollbackInProgress = "in-progress" + +// RestoreAdoptedProp marks a definition that a retry adopted as the leftover of +// an earlier attempt at the same clone or restore, before it wrote anything +// into it. The attempt that created the definition reads it before rolling +// back, and leaves the definition to the retry that answered for it. +// +// It stays on the definition once the adopting request has answered. That +// request cannot know whether the attempt it adopted from is still running, +// and a rollback that started after the mark came off would read no adoption +// and delete what the retry answered for, which is the race the mark closes. +// Kept, it says only that a retry answered for the definition, which stays +// true; it never travels to a snapshot or a definition copied from this one +// (TravellingProps), and it goes with the definition. +const RestoreAdoptedProp = "Blockstor/RestoreAdopted" + +// RestoreVolumesProp records, on a restored or cloned definition, the volumes +// the snapshot it was restored from held, as `=` pairs. +// +// The snapshot is the reference a retry judges a leftover against, and it does +// not always outlive the leftover: an operator can delete it. Judged without +// it, a definition with one volume and one live replica read as finished +// whatever the snapshot held, so a clone that had restored only some of its +// volumes before the snapshot went was reported complete. The record is +// written with the marker, and it is the reference whenever the caller does +// not hand the snapshot in, whether or not the snapshot still exists: every +// rollback's spare-or-delete judgement (ReadsAsFinished) reads it. A leftover +// written before the record existed carries none and is judged as before, on +// its own volumes alone. +const RestoreVolumesProp = "Blockstor/RestoreVolumes" + +// WithRestoreMarker stamps a definition being restored from snap with the +// marker the satellite restores from, and the record of the volumes snap held +// beside it, so every door writes the two together. It returns the props, +// allocated when nil. +func WithRestoreMarker(props map[string]string, snap *apiv1.Snapshot) map[string]string { + if props == nil { + props = map[string]string{} + } + + props[RestoreFromSnapshotProp] = snap.ResourceName + ":" + snap.Name + props[RestoreVolumesProp] = EncodeRestoreVolumes(snap.VolumeDefinitions) + + return props +} + +// EncodeRestoreVolumes writes the RestoreVolumesProp value for a snapshot's +// volumes. +func EncodeRestoreVolumes(vols []apiv1.SnapshotVolumeDef) string { + parts := make([]string, 0, len(vols)) + for _, vol := range vols { + parts = append(parts, strconv.FormatInt(int64(vol.VolumeNumber), 10)+"="+strconv.FormatInt(vol.SizeKib, 10)) + } + + return strings.Join(parts, ",") +} + +// RecordedRestoreVolumes reads the RestoreVolumesProp back. ok is false when +// the definition carries no record, or one that does not parse, which is +// judged as no record at all. +func RecordedRestoreVolumes(props map[string]string) ([]apiv1.SnapshotVolumeDef, bool) { + raw, present := props[RestoreVolumesProp] + if !present { + return nil, false + } + + if raw == "" { + return []apiv1.SnapshotVolumeDef{}, true + } + + pairs := strings.Split(raw, ",") + out := make([]apiv1.SnapshotVolumeDef, 0, len(pairs)) + + for _, pair := range pairs { + number, size, found := strings.Cut(pair, "=") + if !found { + return nil, false + } + + n, err := strconv.ParseInt(number, 10, 32) + if err != nil { + return nil, false + } + + kib, err := strconv.ParseInt(size, 10, 64) + if err != nil { + return nil, false + } + + out = append(out, apiv1.SnapshotVolumeDef{VolumeNumber: int32(n), SizeKib: kib}) + } + + return out, true +} // TravellingProps copies a props bag for use on another object, leaving out // the props that describe the object they were read from and nothing derived @@ -50,7 +157,122 @@ const restoreFromSnapshotMarker = "BlockstorRestoreFromSnapshot" func TravellingProps(props map[string]string) map[string]string { out := maps.Clone(props) delete(out, RollbackAbandonedProp) - delete(out, restoreFromSnapshotMarker) + delete(out, RestoreAdoptedProp) + delete(out, RestoreFromSnapshotProp) + delete(out, RestoreVolumesProp) return out } + +// serverOwnedDefinitionProps are the props blockstor writes onto a definition +// and reads back to decide what that definition is. A caller's prop edit that +// rewrites or removes one of them changes what blockstor believes about the +// definition without blockstor having done anything. +var serverOwnedDefinitionProps = []string{ //nolint:gochecknoglobals // a fixed list, read only + RestoreFromSnapshotProp, + RollbackAbandonedProp, + RestoreAdoptedProp, + RestoreVolumesProp, +} + +// The steps a rollback gives up at before it touches anything: it found a +// snapshot on the definition, or could not read whether there is one, and a +// snapshot is somebody's data it does not destroy. The definition they mark +// is whole. +const ( + RollbackStepSnapshots = "snapshots" + RollbackStepReadSnapshots = "read-snapshots" +) + +// The first steps that take a definition apart, one per door: the REST +// rollback starts by reaping replicas, the CLI one by deleting the definition. +// A rollback writes one of them before it runs it, so a mark that names +// either may stand over a definition already part torn. +const ( + RollbackStepReapReplicas = "reap-replicas" + RollbackStepDeleteDefinition = "delete-definition" +) + +// ErrRollbackStepMarkKept refuses an operator's delete of a rollback mark that +// names a step a rollback gave up at after it had started taking the +// definition apart: clearing the mark hands the remains to a retry that +// resumes over them. A mark over a definition left whole (in progress, or a +// rollback that stopped at its snapshots) is the operator's to clear. +var ErrRollbackStepMarkKept = errors.New("the rollback mark names the step a rollback gave up at " + + "after it had started taking the definition apart; delete the definition instead") + +// RollbackMarkClearRefusal judges an operator's edit by the props before and +// after it: a delete of RollbackAbandonedProp is allowed only when it held a +// mark over a definition left whole. See ServerOwnedPropEditByOperator. +func RollbackMarkClearRefusal(before, after map[string]string) error { + switch before[RollbackAbandonedProp] { + case "", RollbackInProgress, RollbackStepSnapshots, RollbackStepReadSnapshots: + return nil + } + + if _, kept := after[RollbackAbandonedProp]; kept { + return nil + } + + return ErrRollbackStepMarkKept +} + +// ServerOwnedPropEditByOperator is ServerOwnedPropEdit for an operator's own +// edit of a definition (`rd modify`, `rd set-property`), which may delete +// RollbackAbandonedProp by name, while it marks a definition left whole (see +// RollbackMarkClearRefusal). A rollback that had to leave the definition +// whole and then could not take its mark back off leaves one that every retry +// refuses, and deleting the definition is no way out when it holds a volume a +// caller was told exists. Setting the mark, or any edit of the others, stays +// refused. The clone door keeps the strict check: its edits are re-applied by +// every replay, where deleting the mark would clear a rollback still running. +// +// An override of the mark to an empty value is the upstream spelling of the +// same delete (`set-property KEY ""`), and is read as one. +func ServerOwnedPropEditByOperator(overrideProps map[string]string, deleteProps, deleteNamespaces []string) string { + kept := slices.DeleteFunc(slices.Clone(deleteProps), func(k string) bool { return k == RollbackAbandonedProp }) + + overrides := overrideProps + if v, set := overrideProps[RollbackAbandonedProp]; set && v == "" { + overrides = maps.Clone(overrideProps) + delete(overrides, RollbackAbandonedProp) + } + + return ServerOwnedPropEdit(overrides, kept, deleteNamespaces) +} + +// ServerOwnedPropEdit names the first server-owned definition prop a set of +// prop edits would rewrite or remove, or "" when it touches none: a key set or +// deleted by name, or a deleted namespace the key lives in. +func ServerOwnedPropEdit(overrideProps map[string]string, deleteProps, deleteNamespaces []string) string { + for _, key := range serverOwnedDefinitionProps { + if _, set := overrideProps[key]; set { + return key + } + + if slices.Contains(deleteProps, key) { + return key + } + + for _, ns := range deleteNamespaces { + if NamespaceCovers(ns, key) { + return key + } + } + } + + return "" +} + +// NamespaceCovers reports whether deleting namespace ns removes key. It is the +// one reading of a `delete_namespaces` entry, shared by the deleter and every +// guard in front of it: a guard that spelled the namespace another way would +// let through a delete it means to refuse, and then refuse putting the key +// back. A namespace covers the keys below it and not a key spelled as it, as +// upstream's PropsContainer.removeNamespace does; trailing slashes are the +// same namespace, and one that is nothing but slashes names none. +func NamespaceCovers(ns, key string) bool { + ns = strings.TrimRight(ns, "/") + + return ns != "" && strings.HasPrefix(key, ns+"/") +} diff --git a/pkg/store/namespace_covers_test.go b/pkg/store/namespace_covers_test.go new file mode 100644 index 00000000..43aef20f --- /dev/null +++ b/pkg/store/namespace_covers_test.go @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store_test + +import ( + "testing" + + "github.com/cozystack/blockstor/pkg/store" +) + +func TestNamespaceCovers(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + ns, key string + want bool + }{ + {"DrbdOptions", "DrbdOptions/Net/protocol", true}, + {"DrbdOptions/", "DrbdOptions/Net/protocol", true}, + {"DrbdOptions//", "DrbdOptions/Net/protocol", true}, + {"DrbdOptions", "DrbdOptions", false}, + {"DrbdOptions/", "DrbdOptions", false}, + {"DrbdOptions", "DrbdOptionsOther", false}, + {"", "DrbdOptions/Net/protocol", false}, + {"/", "DrbdOptions/Net/protocol", false}, + } { + if got := store.NamespaceCovers(tc.ns, tc.key); got != tc.want { + t.Errorf("NamespaceCovers(%q, %q) = %v, want %v", tc.ns, tc.key, got, tc.want) + } + } +} + +// The server-owned guard reads a namespace the way the deleter does: it +// refuses one the key lives in, in every spelling, and lets through a key +// spelled as a namespace, which the deleter leaves alone. +func TestServerOwnedPropEditReadsNamespacesAsTheDeleterDoes(t *testing.T) { + t.Parallel() + + for _, ns := range []string{"Blockstor", "Blockstor/", "Blockstor//"} { + if got := store.ServerOwnedPropEdit(nil, nil, []string{ns}); got == "" { + t.Errorf("delete_namespaces=[%q] over the server-owned props was let through", ns) + } + } + + if got := store.ServerOwnedPropEdit(nil, nil, []string{store.RestoreFromSnapshotProp}); got != "" { + t.Errorf("a namespace spelled as the key %q was refused as %q; the deleter does not remove it", + store.RestoreFromSnapshotProp, got) + } +} diff --git a/pkg/store/restore_faults_test.go b/pkg/store/restore_faults_test.go new file mode 100644 index 00000000..41bde538 --- /dev/null +++ b/pkg/store/restore_faults_test.go @@ -0,0 +1,483 @@ +// SPDX-License-Identifier: Apache-2.0 + +package store_test + +import ( + "context" + "errors" + "strings" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +var errInjected = errors.New("injected store failure") + +// faults names the store calls a faultStore fails. Each is a read or a write a +// restore decision rests on; a nil field passes the call through. +type faults struct { + rdGet, rdGetUncached, rdPatch error + vdList error + replicaList error + rgGet error +} + +// faultStore fails the calls its faults name and passes everything else to an +// in-memory backend, which has no cache, so a listing and its uncached twin +// read the same copy. +type faultStore struct { + store.Store + + f faults +} + +func (s faultStore) ResourceDefinitions() store.ResourceDefinitionStore { + return faultRDs{ResourceDefinitionStore: s.Store.ResourceDefinitions(), f: s.f} +} + +func (s faultStore) VolumeDefinitions() store.VolumeDefinitionStore { + return faultVDs{VolumeDefinitionStore: s.Store.VolumeDefinitions(), f: s.f} +} + +func (s faultStore) Resources() store.ResourceStore { + return faultReplicas{ResourceStore: s.Store.Resources(), f: s.f} +} + +func (s faultStore) ResourceGroups() store.ResourceGroupStore { + return faultRGs{ResourceGroupStore: s.Store.ResourceGroups(), f: s.f} +} + +type faultRDs struct { + store.ResourceDefinitionStore + + f faults +} + +func (r faultRDs) Get(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + if r.f.rdGet != nil { + return apiv1.ResourceDefinition{}, r.f.rdGet + } + + return r.ResourceDefinitionStore.Get(ctx, name) //nolint:wrapcheck // pass-through test double +} + +func (r faultRDs) GetUncached(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { + if r.f.rdGetUncached != nil { + return apiv1.ResourceDefinition{}, r.f.rdGetUncached + } + + return r.ResourceDefinitionStore.GetUncached(ctx, name) //nolint:wrapcheck // pass-through test double +} + +func (r faultRDs) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + if r.f.rdPatch != nil { + return r.f.rdPatch + } + + return r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) //nolint:wrapcheck // pass-through test double +} + +type faultVDs struct { + store.VolumeDefinitionStore + + f faults +} + +func (v faultVDs) List(ctx context.Context, rdName string) ([]apiv1.VolumeDefinition, error) { + if v.f.vdList != nil { + return nil, v.f.vdList + } + + return v.VolumeDefinitionStore.List(ctx, rdName) //nolint:wrapcheck // pass-through test double +} + +func (v faultVDs) ListUncached(ctx context.Context, rdName string) ([]apiv1.VolumeDefinition, error) { + return v.List(ctx, rdName) +} + +type faultReplicas struct { + store.ResourceStore + + f faults +} + +func (r faultReplicas) ListByDefinition(ctx context.Context, rdName string) ([]apiv1.Resource, error) { + if r.f.replicaList != nil { + return nil, r.f.replicaList + } + + return r.ResourceStore.ListByDefinition(ctx, rdName) //nolint:wrapcheck // pass-through test double +} + +func (r faultReplicas) ListByDefinitionUncached(ctx context.Context, rdName string) ([]apiv1.Resource, error) { + return r.ListByDefinition(ctx, rdName) +} + +type faultRGs struct { + store.ResourceGroupStore + + f faults +} + +func (g faultRGs) Get(ctx context.Context, name string) (apiv1.ResourceGroup, error) { + if g.f.rgGet != nil { + return apiv1.ResourceGroup{}, g.f.rgGet + } + + return g.ResourceGroupStore.Get(ctx, name) //nolint:wrapcheck // pass-through test double +} + +// assertUntouched fails when a decision that returned an error still wrote to +// the target: a mark, or a replica. +func assertUntouched(t *testing.T, backend store.Store, rdName string) { + t.Helper() + + rd, err := backend.ResourceDefinitions().Get(t.Context(), rdName) + if err != nil { + t.Fatalf("the target is gone: %v", err) + } + + for _, key := range []string{store.RestoreAdoptedProp, store.RollbackAbandonedProp} { + if v, ok := rd.Props[key]; ok { + t.Errorf("a refused decision wrote %s=%q", key, v) + } + } + + replicas, err := backend.Resources().ListByDefinition(t.Context(), rdName) + if err != nil { + t.Fatalf("list the replicas: %v", err) + } + + if len(replicas) != 0 { + t.Errorf("a refused decision placed %d replica(s)", len(replicas)) + } +} + +// A prepared target is judged on four reads. Any that fails is an error, never +// a verdict: "not prepared" refuses a valid restore, and "prepared" marks a +// definition nobody looked at. +func TestPreparedRestoreTargetFailsClosedOnAFailedRead(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + f faults + }{ + {name: "definition", f: faults{rdGetUncached: errInjected}}, + {name: "volumes", f: faults{vdList: errInjected}}, + {name: "replicas", f: faults{replicaList: errInjected}}, + {name: "source", f: faults{rdGet: errInjected}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + snap := seedPreparedTarget(t, backend) + + rd, err := backend.ResourceDefinitions().Get(t.Context(), "pvc-dst") + if err != nil { + t.Fatalf("read the target: %v", err) + } + + prepared, err := store.PreparedRestoreTarget(t.Context(), faultStore{backend, tc.f}, &rd, snap) + if prepared || !errors.Is(err, errInjected) { + t.Errorf("judged (%v, %v) over a failed read, want (false, the read's error)", prepared, err) + } + + assertUntouched(t, backend, "pvc-dst") + }) + } +} + +// Replicas that are all going away are refused with a reason, not passed over +// as "not prepared", which reads to the caller as somebody else's definition. +func TestPreparedRestoreTargetRefusesATargetBeingTornDown(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + snap := seedPreparedTarget(t, backend) + + if err := backend.Resources().Create(t.Context(), &apiv1.Resource{ + Name: "pvc-dst", NodeName: "node-a", Flags: []string{apiv1.ResourceFlagDelete}, + }); err != nil { + t.Fatalf("seed the terminating replica: %v", err) + } + + rd, err := backend.ResourceDefinitions().Get(t.Context(), "pvc-dst") + if err != nil { + t.Fatalf("read the target: %v", err) + } + + prepared, err := store.PreparedRestoreTarget(t.Context(), backend, &rd, snap) + if prepared || !errors.Is(err, store.ErrRestoreTargetTearingDown) { + t.Errorf("judged (%v, %v), want (false, ErrRestoreTargetTearingDown)", prepared, err) + } +} + +// markTarget puts this restore's marker, and whatever else the case needs, on +// the seeded target. +func markTarget(t *testing.T, backend store.Store, group string, flags []string) { + t.Helper() + + if err := backend.ResourceDefinitions().PatchResourceDefinitionSpec(t.Context(), "pvc-dst", + func(rd *apiv1.ResourceDefinition) error { + rd.Props = marked() + rd.ResourceGroupName = group + rd.Flags = flags + + return nil + }); err != nil { + t.Fatalf("mark the target: %v", err) + } +} + +// A leftover is judged on reads too, and a failed one is an error the door +// stops on, never a verdict it acts on. +func TestJudgeRestoreLeftoverFailsClosedOnAFailedRead(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + group string + f faults + }{ + {name: "definition", f: faults{rdGetUncached: errInjected}}, + {name: "volumes", f: faults{vdList: errInjected}}, + {name: "replicas", f: faults{replicaList: errInjected}}, + {name: "group", group: "grp", f: faults{rgGet: errInjected}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + snap := seedPreparedTarget(t, backend) + + if err := backend.ResourceGroups().Create(t.Context(), &apiv1.ResourceGroup{Name: "grp"}); err != nil { + t.Fatalf("seed the group: %v", err) + } + + markTarget(t, backend, tc.group, nil) + + _, _, err := store.JudgeRestoreLeftover(t.Context(), faultStore{backend, tc.f}, "pvc-dst", snap) + if !errors.Is(err, errInjected) { + t.Errorf("judged over a failed read with error %v, want the read's error", err) + } + + assertUntouched(t, backend, "pvc-dst") + }) + } +} + +// The DELETE flag on the definition itself is a tear-down, refused before the +// leftover is assessed, and a volume the snapshot never recorded makes the +// leftover somebody else's. +func TestJudgeRestoreLeftoverRefusesADeletedOrForeignLeftover(t *testing.T) { + t.Parallel() + + t.Run("deleting", func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + snap := seedPreparedTarget(t, backend) + markTarget(t, backend, "", []string{apiv1.ResourceFlagDelete}) + + got, own, err := store.JudgeRestoreLeftover(t.Context(), backend, "pvc-dst", snap) + if !own || got != store.LeftoverTearingDown || !errors.Is(err, store.ErrRestoreTargetTearingDown) { + t.Errorf("judged (%v, %v, %v), want (TearingDown, true, ErrRestoreTargetTearingDown)", got, own, err) + } + }) + + t.Run("foreign", func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + snap := seedPreparedTarget(t, backend) + markTarget(t, backend, "", nil) + + if err := backend.VolumeDefinitions().Create(t.Context(), "pvc-dst", + &apiv1.VolumeDefinition{VolumeNumber: 5, SizeKib: 1 << 20}); err != nil { + t.Fatalf("seed the foreign volume: %v", err) + } + + got, own, err := store.JudgeRestoreLeftover(t.Context(), backend, "pvc-dst", snap) + if !own || got != store.LeftoverForeign || !errors.Is(err, store.ErrRestoreTargetForeign) { + t.Errorf("judged (%v, %v, %v), want (Foreign, true, ErrRestoreTargetForeign)", got, own, err) + } + }) + + // The REST door refuses a foreign volume before it asks about the group + // or a rollback mark, and the CLI asks in the same order, so the two + // word one leftover the same way. + t.Run("foreign-before-group-and-mark", func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + snap := seedPreparedTarget(t, backend) + markTarget(t, backend, "grp-gone", nil) + + if err := backend.ResourceDefinitions().PatchResourceDefinitionSpec(t.Context(), "pvc-dst", + func(rd *apiv1.ResourceDefinition) error { + rd.Props[store.RollbackAbandonedProp] = store.RollbackInProgress + + return nil + }); err != nil { + t.Fatalf("mark a rollback in progress: %v", err) + } + + if err := backend.VolumeDefinitions().Create(t.Context(), "pvc-dst", + &apiv1.VolumeDefinition{VolumeNumber: 5, SizeKib: 1 << 20}); err != nil { + t.Fatalf("seed the foreign volume: %v", err) + } + + _, _, err := store.JudgeRestoreLeftover(t.Context(), backend, "pvc-dst", snap) + if !errors.Is(err, store.ErrRestoreTargetForeign) { + t.Errorf("judged %v, want ErrRestoreTargetForeign ahead of the group and the mark", err) + } + }) +} + +// The replica count a leftover is judged on is an error when the listing +// fails, never zero replicas. +func TestCountReplicasReportsAFailedListing(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedPreparedTarget(t, backend) + + if _, err := store.CountReplicas(t.Context(), faultStore{backend, faults{replicaList: errInjected}}, + "pvc-dst"); !errors.Is(err, errInjected) { + t.Errorf("CountReplicas over a failed listing = %v, want the listing's error", err) + } +} + +// The adopting half of the handshake must not go on when its own mark did not +// land: the creator's rollback would not see it and would delete what the +// adopter then answers for. +func TestClaimAdoptedLeftoverFailsWhenTheMarkDoesNotLand(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedPreparedTarget(t, backend) + + if err := store.ClaimAdoptedLeftover(t.Context(), faultStore{backend, faults{rdPatch: errInjected}}, + "pvc-dst", &apiv1.Snapshot{}); !errors.Is(err, errInjected) { + t.Errorf("claim over a refused mark = %v, want the patch's error", err) + } +} + +// A rollback mark that cannot be cleared is reported: the caller tells the +// operator the definition still carries it, and a retry would meet it. +func TestReleaseRollbackReportsAMarkItCouldNotClear(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedPreparedTarget(t, backend) + + if err := store.ReleaseRollback(t.Context(), faultStore{backend, faults{rdPatch: errInjected}}, + "pvc-dst"); !errors.Is(err, errInjected) { + t.Errorf("release over a refused patch = %v, want the patch's error", err) + } +} + +// A rollback that stopped at the definition's snapshots touched nothing, and +// `rd d` refuses the definition over them, so the CLI names the clear as the +// way to keep it, as the REST door does, rather than only a delete that is +// refused. +func TestJudgeRestoreLeftoverOverASnapshotsMarkNamesTheClear(t *testing.T) { + t.Parallel() + + for _, step := range []string{store.RollbackStepSnapshots, store.RollbackStepReadSnapshots} { + t.Run(step, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + snap := seedPreparedTarget(t, backend) + markTarget(t, backend, "", nil) + + if err := backend.ResourceDefinitions().PatchResourceDefinitionSpec(t.Context(), "pvc-dst", + func(rd *apiv1.ResourceDefinition) error { + rd.Props[store.RollbackAbandonedProp] = step + + return nil + }); err != nil { + t.Fatalf("mark the rollback's step: %v", err) + } + + _, _, err := store.JudgeRestoreLeftover(t.Context(), backend, "pvc-dst", snap) + if !errors.Is(err, store.ErrRestoreRollbackAbandoned) || + !strings.Contains(err.Error(), "set-property pvc-dst "+store.RollbackAbandonedProp) { + t.Errorf("judged %v, want the abandoned-rollback refusal naming the clear", err) + } + }) + } +} + +// Once the snapshot is gone, a leftover the CLI refuses is not told to +// "restore again": deleted, it would meet a snapshot that is no longer there. +// It names another snapshot, as the tear-down refusal does. +func TestJudgeRestoreLeftoverWithTheSnapshotGoneNamesAnotherSnapshot(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + mark string + }{ + {name: "foreign"}, + {name: "rollback-gave-up", mark: "reap-replicas"}, + {name: "rollback-in-progress", mark: store.RollbackInProgress}, + {name: "rollback-stopped-at-snapshots", mark: store.RollbackStepSnapshots}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + snap := seedPreparedTarget(t, backend) + markTarget(t, backend, "", nil) + + // The marker as every door writes it, with the record of what + // the snapshot held: the leftover is judged against it once the + // snapshot is gone. + if err := backend.ResourceDefinitions().PatchResourceDefinitionSpec(t.Context(), "pvc-dst", + func(rd *apiv1.ResourceDefinition) error { + rd.Props = store.WithRestoreMarker(rd.Props, snap) + + return nil + }); err != nil { + t.Fatalf("record the snapshot's volumes: %v", err) + } + + if tc.mark != "" { + if err := backend.ResourceDefinitions().PatchResourceDefinitionSpec(t.Context(), "pvc-dst", + func(rd *apiv1.ResourceDefinition) error { + rd.Props[store.RollbackAbandonedProp] = tc.mark + + return nil + }); err != nil { + t.Fatalf("mark the rollback's step: %v", err) + } + } else if err := backend.VolumeDefinitions().Create(t.Context(), "pvc-dst", + &apiv1.VolumeDefinition{VolumeNumber: 5, SizeKib: 1 << 20}); err != nil { + t.Fatalf("seed the foreign volume: %v", err) + } + + _, _, err := store.JudgeRestoreLeftoverOf(t.Context(), backend, "pvc-dst", snap.ResourceName, snap.Name, nil) + if err == nil { + t.Fatal("judged nil, want a refusal") + } + + if !strings.Contains(err.Error(), "another snapshot") || strings.Contains(err.Error(), "restore again") { + t.Errorf("judged %v, want a refusal naming another snapshot and not %q", err, "restore again") + } + + // The arms that name the clear name the definition in it, not the + // snapshot or the way back to a restore. + if (tc.mark == store.RollbackInProgress || tc.mark == store.RollbackStepSnapshots) && + !strings.Contains(err.Error(), "set-property pvc-dst "+store.RollbackAbandonedProp) { + t.Errorf("judged %v, want the clear named for pvc-dst", err) + } + }) + } +} diff --git a/pkg/store/restore_target.go b/pkg/store/restore_target.go new file mode 100644 index 00000000..04dd573b --- /dev/null +++ b/pkg/store/restore_target.go @@ -0,0 +1,740 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "slices" + "strings" + "time" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" +) + +// ErrRestoreTargetTaken reports a definition that carries a restore marker +// naming another snapshot. +var ErrRestoreTargetTaken = errors.New("resource definition is already a restore of another snapshot") + +// ErrRestoreTargetLayers reports a prepared target whose layer stack is not +// the snapshot source's. +var ErrRestoreTargetLayers = errors.New("prepared restore target has a different layer stack than the snapshot's source") + +// ErrRestoreTargetTearingDown reports a prepared target whose every replica is +// still being deleted. +var ErrRestoreTargetTearingDown = errors.New("prepared restore target still has replicas being deleted") + +// uncachedVolumeLister lists a definition's volumes from the API server. +type uncachedVolumeLister interface { + ListUncached(ctx context.Context, rdName string) ([]apiv1.VolumeDefinition, error) +} + +// uncachedReplicaLister lists a definition's replicas from the API server. +type uncachedReplicaLister interface { + ListByDefinitionUncached(ctx context.Context, rdName string) ([]apiv1.Resource, error) +} + +// LiveVolumes reads a definition's volumes past the cache where the store can; +// a store without a cache answers the same question from its only copy. +func LiveVolumes(ctx context.Context, st Store, rdName string) ([]apiv1.VolumeDefinition, error) { + vds := st.VolumeDefinitions() + if direct, ok := vds.(uncachedVolumeLister); ok { + return direct.ListUncached(ctx, rdName) //nolint:wrapcheck // the caller wraps with the definition + } + + return vds.List(ctx, rdName) //nolint:wrapcheck // the caller wraps with the definition +} + +// GetResourceUncached reads one replica from the API server where the store +// has a direct reader, and through the store otherwise. A decision on whether +// a replica a create collided with is live, or already being deleted, has to +// see the object as it is now: a cache that trails shows a replica accepted +// for deletion as a live one. +func GetResourceUncached(ctx context.Context, st Store, rdName, node string) (apiv1.Resource, error) { + resources := st.Resources() + if direct, ok := resources.(interface { + GetUncached(ctx context.Context, rdName, node string) (apiv1.Resource, error) + }); ok { + return direct.GetUncached(ctx, rdName, node) //nolint:wrapcheck // the store wraps with the replica it names + } + + return resources.Get(ctx, rdName, node) //nolint:wrapcheck // the store wraps with the replica it names +} + +// LiveReplicas reads a definition's replicas past the cache where the store can. +func LiveReplicas(ctx context.Context, st Store, rdName string) ([]apiv1.Resource, error) { + resources := st.Resources() + if direct, ok := resources.(uncachedReplicaLister); ok { + return direct.ListByDefinitionUncached(ctx, rdName) //nolint:wrapcheck // the caller wraps with the definition + } + + return resources.ListByDefinition(ctx, rdName) //nolint:wrapcheck // the caller wraps with the definition +} + +// PreparedRestoreTarget reports whether a definition with no restore marker is +// the target LINSTOR's own restore sequence prepares for snapshot: created +// empty, its volume definitions restored from that snapshot, and no replica +// holding data yet. +// +// That is the documented flow (`rd c`, `s vd restore`, `s rsc restore`) and the +// one linstor-csi runs for every volume restored from a snapshot: VolFromSnap +// creates the definition itself, restores the volume definitions only when the +// definition has none, and then asks for the resources. The definition it made +// carries no marker, and has to be taken as this restore's target rather than +// as somebody else's. +// +// Only that state is taken. A definition already marked is judged by its +// marker; one being deleted, one whose volumes are not exactly the snapshot's, +// and one with a live replica may hold somebody's data, and are not prepared. +// +// Two states are prepared in every other respect and still refused, with an +// error saying why, before anything is written to the definition: +// +// - A layer stack that is not the source's (ErrRestoreTargetLayers). The +// restore puts the source's bytes under the target's stack, so a layer the +// source did not have writes its own metadata across them (LUKS formats), +// and one it had is missing on read (the consumer sees ciphertext). The +// create path inherits the source's stack and the clone refuses a changed +// set for the same reason; through linstor-csi this is a VolumeSnapshot +// restored into a PVC of another StorageClass. +// - Replicas that are all still being deleted (ErrRestoreTargetTearingDown). +// Placing over them races the tear-down, and a retry once it finished +// takes the target cleanly. +// +// The definition, its volumes and its replicas are all read from the API +// server. linstor-csi's volume restore and the resource restore after it can +// land on different replicas of this server, and a cache on the second that +// has not seen the first's writes answers both questions wrong. +func PreparedRestoreTarget(ctx context.Context, st Store, cached *apiv1.ResourceDefinition, snap *apiv1.Snapshot) (bool, error) { + live, err := st.ResourceDefinitions().GetUncached(ctx, cached.Name) + if errors.Is(err, ErrNotFound) { + return false, nil + } + + if err != nil { + return false, fmt.Errorf("read %q: %w", cached.Name, err) + } + + rd := &live + + if rd.Props[RestoreFromSnapshotProp] != "" || slices.Contains(rd.Flags, apiv1.ResourceFlagDelete) { + return false, nil + } + + vds, err := LiveVolumes(ctx, st, rd.Name) + if err != nil { + return false, fmt.Errorf("list the volumes of %q: %w", rd.Name, err) + } + + if !sameVolumes(vds, snap.VolumeDefinitions) { + return false, nil + } + + replicas, err := LiveReplicas(ctx, st, rd.Name) + if err != nil { + return false, fmt.Errorf("list the replicas of %q: %w", rd.Name, err) + } + + for i := range replicas { + if !slices.Contains(replicas[i].Flags, apiv1.ResourceFlagDelete) { + return false, nil + } + } + + if len(replicas) > 0 { + return false, fmt.Errorf("%w: %q", ErrRestoreTargetTearingDown, rd.Name) + } + + source, err := st.ResourceDefinitions().Get(ctx, snap.ResourceName) + if err != nil { + return false, fmt.Errorf("read the source %q of snapshot %q: %w", snap.ResourceName, snap.Name, err) + } + + have, want := EffectiveLayerStack(source.LayerStack), EffectiveLayerStack(rd.LayerStack) + if !sameLayerSet(have, want) { + return false, fmt.Errorf("%w: %q has %s, the source %q has %s", ErrRestoreTargetLayers, + rd.Name, strings.Join(want, ","), source.Name, strings.Join(have, ",")) + } + + return true, nil +} + +// EffectiveLayerStack resolves a stored stack the way the data plane does: the +// dispatcher hands the satellite the definition's own stack and nothing else, +// and the satellite brings up DRBD over STORAGE for an empty one. A parent +// group's stack is not part of it: it is copied onto a definition only when +// the definition is created through `rd c`. It returns a copy. +// +// The control plane reads the group's stack for a definition stored without +// one (the quorum witness, the autoplace gate), so a restored or cloned +// definition left empty under a group with another stack is brought up as one +// shape and judged as another. Stamping this onto it keeps the two in step. +func EffectiveLayerStack(stack []string) []string { + if len(stack) == 0 { + return apiv1.DefaultLayerStack() + } + + return append([]string(nil), stack...) +} + +// sameLayerSet compares two stacks as sets. Order follows from the kinds in a +// stack and LINSTOR folds name case, so neither is a difference. +func sameLayerSet(have, want []string) bool { + for _, layer := range have { + if !apiv1.LayerInStack(want, layer) { + return false + } + } + + for _, layer := range want { + if !apiv1.LayerInStack(have, layer) { + return false + } + } + + return true +} + +// sameVolumes reports whether a definition's volumes are exactly the ones the +// snapshot recorded: the same numbers at the same sizes, nothing more. +func sameVolumes(vds []apiv1.VolumeDefinition, recorded []apiv1.SnapshotVolumeDef) bool { + if len(recorded) == 0 || len(vds) != len(recorded) { + return false + } + + sizes := make(map[int32]int64, len(recorded)) + for i := range recorded { + sizes[recorded[i].VolumeNumber] = recorded[i].SizeKib + } + + for i := range vds { + size, ok := sizes[vds[i].VolumeNumber] + if !ok || size != vds[i].SizeKib { + return false + } + } + + return true +} + +// stampRestoreMarker writes the restore marker for snap onto a prepared target, +// reporting whether this call wrote it, as opposed to finding the same one +// already there. It is a patch, so a second request racing this one over the +// same target writes the same value; a marker naming another snapshot that +// landed in between is ErrRestoreTargetTaken, and it is left as it is. +func stampRestoreMarker(ctx context.Context, st Store, rdName string, snap *apiv1.Snapshot) (bool, error) { + marker := snap.ResourceName + ":" + snap.Name + + var wrote bool + + err := st.ResourceDefinitions().PatchResourceDefinitionSpec(ctx, rdName, func(rd *apiv1.ResourceDefinition) error { + current := rd.Props[RestoreFromSnapshotProp] + if current != "" && !strings.EqualFold(current, marker) { + return fmt.Errorf("%w: %q carries %q", ErrRestoreTargetTaken, rdName, current) + } + + // The mutate runs again on a conflict, so the answer is the last + // run's, against the state the patch actually applied to. + wrote = current == "" + + rd.Props = WithRestoreMarker(rd.Props, snap) + rd.LayerStack = EffectiveLayerStack(rd.LayerStack) + + return nil + }) + if err != nil { + return false, fmt.Errorf("stamp the restore marker on %q: %w", rdName, err) + } + + return wrote, nil +} + +// AdoptPreparedRestoreTarget marks a target PreparedRestoreTarget judged +// prepared as this restore's. It reports whether this call is the one that +// marked it: false means the same marker was already there, put by another +// request for the same restore, so the caller judges the target as that +// request's leftover, the way any retry is judged. +// +// Once the marker is on, it stays. linstor-csi re-issues CreateVolume on a +// timeout, and the retry finds the marker this call just wrote, resumes and +// places the replica; this call then meets that replica as AlreadyExists when +// it places its own, which the placement already takes as placed. Taking the +// marker back off on seeing a replica would leave a definition CSI was told is +// restored with no marker, so every later replica would come up blank and every +// later call would be refused. +// +// The window this leaves is a replica placed by hand between the judgement and +// the marker: it is treated as part of the restore. On a target the caller +// prepared for exactly this restore nothing else places one in that moment, and +// a replica placed any later lands on a definition already marked as a restore, +// the state any writer racing a restore on its own target meets. +func AdoptPreparedRestoreTarget(ctx context.Context, st Store, rdName string, snap *apiv1.Snapshot) (bool, error) { + return stampRestoreMarker(ctx, st, rdName, snap) +} + +// ErrRestoreTargetGroupGone reports a restore leftover whose parent resource +// group no longer exists. +var ErrRestoreTargetGroupGone = errors.New("restore leftover is parented to a resource group that no longer exists") + +// ErrRestoreRollbackAbandoned reports a restore leftover an earlier attempt +// left when its rollback gave up. +var ErrRestoreRollbackAbandoned = errors.New("restore leftover is what an earlier attempt left when its rollback gave up") + +// ErrRestoreTargetForeign reports a restore leftover holding a volume the +// restore would not have written. +var ErrRestoreTargetForeign = errors.New("restore leftover holds a volume this restore would not have written") + +// JudgeRestoreLeftover is the CLI's door onto the judgement the REST restore +// makes over a definition that already carries this restore's marker, so the +// two judge a leftover the same way: neither re-places what the other leaves +// alone, nor finishes what the other refuses. What each door then places +// still differs where the doors themselves do: the CLI always places replicas, +// while a bare REST restore places none. +// +// It returns (progress, true, nil) for this restore's own leftover, and +// (_, false, nil) for anything else under the name, which is not this +// restore's to judge. The gates run in the order the REST door runs them: +// a tear-down (the DELETE flag, or every replica being deleted), then a +// volume the restore would not have written, then a parent group that is +// gone, then a mark left by a rollback that gave up. +// Past them the leftover is judged by store.AssessLeftover: finished means the +// snapshot's volumes and at least one replica with a disk (HoldsData), and is +// not re-placed. +// Re-placing it re-stamped a replica on a node the operator had emptied, +// restored from the point-in-time beside one that had moved on with live +// writes. +func JudgeRestoreLeftover( + ctx context.Context, st Store, rdName string, snap *apiv1.Snapshot, +) (LeftoverProgress, bool, error) { + return JudgeRestoreLeftoverOf(ctx, st, rdName, snap.ResourceName, snap.Name, snap) +} + +// JudgeRestoreLeftoverOf is JudgeRestoreLeftover for a restore named by its +// source and snapshot, with snap nil once the snapshot is gone: the leftover +// is then judged against the volumes the definition recorded when it was +// restored (RestoreVolumesProp), since an absent snapshot cannot unmake a +// finished target. +func JudgeRestoreLeftoverOf( + ctx context.Context, st Store, rdName, srcRD, snapName string, snap *apiv1.Snapshot, +) (LeftoverProgress, bool, error) { + rd, err := st.ResourceDefinitions().GetUncached(ctx, rdName) + if errors.Is(err, ErrNotFound) { + return LeftoverUnfinished, false, nil + } + + if err != nil { + return LeftoverUnfinished, false, fmt.Errorf("read %q: %w", rdName, err) + } + + marker := srcRD + ":" + snapName + if !strings.EqualFold(rd.Props[RestoreFromSnapshotProp], marker) { + return LeftoverUnfinished, false, nil + } + + if slices.Contains(rd.Flags, apiv1.ResourceFlagDelete) { + return LeftoverTearingDown, true, fmt.Errorf("%w: %q is being deleted", ErrRestoreTargetTearingDown, rd.Name) + } + + vds, err := LiveVolumes(ctx, st, rd.Name) + if err != nil { + return LeftoverUnfinished, true, fmt.Errorf("list the volumes of %q: %w", rd.Name, err) + } + + progress, err := AssessLeftover(ctx, st, rd.Name, vds, snap, true) + if err != nil { + return LeftoverUnfinished, true, err + } + + if progress == LeftoverTearingDown { + return progress, true, fmt.Errorf("%w: %q", ErrRestoreTargetTearingDown, rd.Name) + } + + if progress == LeftoverForeign { + return progress, true, fmt.Errorf("%w: delete %q and %s", ErrRestoreTargetForeign, rd.Name, restoreAgainFrom(snap)) + } + + err = RestoreTargetGroupSurvived(ctx, st, &rd) + if err != nil { + return progress, true, err + } + + err = rollbackMarkRefusal(rd.Name, rd.Props[RollbackAbandonedProp], snap) + if err != nil { + return progress, true, err + } + + return progress, true, nil +} + +// rollbackMarkRefusal is the CLI door's refusal over a rollback mark, worded +// for the step it names; nil when there is none. +func rollbackMarkRefusal(rdName, step string, snap *apiv1.Snapshot) error { + again := restoreAgainFrom(snap) + + switch step { + case "": + return nil + case RollbackInProgress: + return fmt.Errorf("%w: %q is being rolled back by the attempt that created it, "+ + "or that attempt stopped mid-way; %s once it has finished, which on the REST door takes under a minute; "+ + "if the mark is still there a minute after the failure, run "+ + "`blockstor resource-definition set-property %s %s` with no value to keep it as it stands, "+ + "or delete it by hand. "+ + "Cleared while the rollback still runs, it stops that rollback before it deletes anything", + ErrRestoreRollbackAbandoned, rdName, again, rdName, RollbackAbandonedProp) + case RollbackStepSnapshots, RollbackStepReadSnapshots: + // Stopped at the definition's snapshots, before touching anything: + // `rd d` refuses it over those snapshots, so the way to keep it is + // the clear, as the REST door says. + return fmt.Errorf("%w (%s): %q is whole, and the rollback stopped at its snapshots; "+ + "delete the snapshots if they are not needed, then delete %q by hand and %s, or run "+ + "`blockstor resource-definition set-property %s %s` with no value to keep it as it stands", + ErrRestoreRollbackAbandoned, step, rdName, rdName, again, rdName, RollbackAbandonedProp) + default: + return fmt.Errorf("%w (%s): delete %q by hand, then %s", + ErrRestoreRollbackAbandoned, step, rdName, again) + } +} + +// restoreAgainFrom is the way back to a restore once a leftover has been +// deleted: the same restore while its snapshot is there, and another snapshot +// once it is gone, since the same one has nothing left to restore from. +func restoreAgainFrom(snap *apiv1.Snapshot) string { + if snap == nil { + return "restore it from another snapshot" + } + + return "restore again" +} + +// RestoreTargetGroupSurvived refuses a restore target whose parent group no +// longer exists. Both doors ask it before they write anything onto a +// definition that is already there, prepared or a leftover: a marker or a +// replica on a definition parented to nothing lists fine and places badly. +func RestoreTargetGroupSurvived(ctx context.Context, st Store, rd *apiv1.ResourceDefinition) error { + if rd.ResourceGroupName == "" { + return nil + } + + _, err := st.ResourceGroups().Get(ctx, rd.ResourceGroupName) + if errors.Is(err, ErrNotFound) { + return fmt.Errorf("%w: %q, group %q", ErrRestoreTargetGroupGone, rd.Name, rd.ResourceGroupName) + } + + if err != nil { + return fmt.Errorf("read resource group %q: %w", rd.ResourceGroupName, err) + } + + return nil +} + +// ErrAdoptedLeftoverRollingBack refuses to adopt a leftover whose creator has +// started rolling it back. +var ErrAdoptedLeftoverRollingBack = errors.New("the attempt that created it is rolling it back") + +// ClaimAdoptedLeftover is the adopting half of the handshake with the attempt +// that created a restore's definition, which may still be running. Both doors +// take it before they write anything onto a leftover. +// +// It is one patch: the mutate refuses a definition that carries the creator's +// rollback mark (RollbackAbandonedProp) and otherwise writes the adoption mark +// (RestoreAdoptedProp), plus the volumes snap holds (RestoreVolumesProp) when +// snap is given. The patch is a compare-and-swap on both backends, and the +// rollback writes its own mark the same way, so the two serialise: a claim +// that lands first is seen by the rollback's later adoption read, which then +// yields; a rollback that lands first is seen by the claim's mutate, which then +// writes nothing. A refused claim therefore leaves nothing behind to release. +// +// The mark is a fresh random token, so a write that answered an error can be +// told from another claim's: one that landed anyway is read back and accepted. +// A write that may have landed and cannot be confirmed is reported as the +// error it was; its mark then stays, which fails closed, since an adoption +// mark only ever makes a rollback leave the definition in place, and the +// client's retry claims it again. A definition deleted and recreated under the +// name before the patch is marked the same way, for the same reason. +func ClaimAdoptedLeftover(ctx context.Context, st Store, rdName string, snap *apiv1.Snapshot) error { + token := newAdoptionToken() + + err := st.ResourceDefinitions().PatchResourceDefinitionSpec(ctx, rdName, + func(rd *apiv1.ResourceDefinition) error { + if rd.Props[RollbackAbandonedProp] != "" { + return ErrAdoptedLeftoverRollingBack + } + + if rd.Props == nil { + rd.Props = map[string]string{} + } + + rd.Props[RestoreAdoptedProp] = token + + if snap != nil { + rd.Props[RestoreVolumesProp] = EncodeRestoreVolumes(snap.VolumeDefinitions) + } + + return nil + }) + if errors.Is(err, ErrAdoptedLeftoverRollingBack) { + return fmt.Errorf("%q cannot be adopted: %w", rdName, ErrAdoptedLeftoverRollingBack) + } + + if err != nil && !claimLanded(ctx, st, rdName, token) { + return fmt.Errorf("mark %q as adopted: %w", rdName, err) + } + + return nil +} + +// ClaimReadBackBudget bounds the read that checks whether a claim whose patch +// answered an error landed anyway. +const ClaimReadBackBudget = 2 * time.Second + +// claimLanded reports whether a claim whose patch answered an error carries +// its own token anyway, as a write at its deadline may. It reads detached from +// ctx: a write that failed at the request's deadline leaves ctx dead, and the +// read that would accept it has to go out regardless. +func claimLanded(ctx context.Context, st Store, rdName, token string) bool { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), ClaimReadBackBudget) + defer cancel() + + current, err := st.ResourceDefinitions().GetUncached(ctx, rdName) + if err != nil { + return false + } + + return current.Props[RestoreAdoptedProp] == token +} + +// newAdoptionToken names one claim. It is random, so two claims in the same +// instant are still two tokens. +func newAdoptionToken() string { + var b [8]byte + + _, _ = rand.Read(b[:]) // crypto/rand.Read does not fail + + return hex.EncodeToString(b[:]) +} + +// ErrRollbackYielded reports a rollback that left the definition alone because +// another request marked it adopted, and may already have answered for it. +var ErrRollbackYielded = errors.New("another request marked the definition adopted, so it was left in place") + +// ErrRollbackAnswered reports a rollback that left the definition alone because +// it already reads as finished. A replay takes the adoption mark before it +// answers, but the clone status poll answers a finished leftover without +// writing anything, so the mark cannot be what tells the creator somebody was +// told it is done. +var ErrRollbackAnswered = errors.New("it already reads as finished, so a retry or a status poll " + + "may have answered for it, and it was left in place") + +// ReadsAsFinished is the judgement a replay or a status poll would make of the +// definition now, with the volumes and replicas read past the cache. It is +// judged against the shape the definition recorded when it was restored +// (RestoreVolumesProp), which is the snapshot's, so it reads no snapshot. +// needReplica is the door's own: a clone and a placed restore need a replica, +// a bare restore does not. +// +// A rollback asks it after its in-progress mark is on. A replay settles this +// through the adoption mark instead; the clone status poll, which writes +// nothing, may have answered for the definition before the mark landed, and is +// seen here because the creator's own writes have stopped and nothing it +// already placed is gone yet. +func ReadsAsFinished(ctx context.Context, st Store, rdName string, needReplica bool) (bool, error) { + rd, err := st.ResourceDefinitions().GetUncached(ctx, rdName) + if errors.Is(err, ErrNotFound) { + return false, nil + } + + if err != nil { + return false, fmt.Errorf("read %q: %w", rdName, err) + } + + // Being deleted is not finished, to the poll or the replay either. + if slices.Contains(rd.Flags, apiv1.ResourceFlagDelete) { + return false, nil + } + + vds, err := LiveVolumes(ctx, st, rdName) + if errors.Is(err, ErrNotFound) { + return false, nil + } + + if err != nil { + return false, fmt.Errorf("list the volumes of %q: %w", rdName, err) + } + + progress, err := AssessLeftover(ctx, st, rdName, vds, nil, needReplica) + if err != nil { + return false, err + } + + return progress == LeftoverFinished, nil +} + +// AdoptedElsewhere reads, past the cache, whether a request has adopted the +// definition (ClaimAdoptedLeftover). A rollback asks it after its own mark has +// landed, so a claim that landed before is seen here and one that comes after +// refuses itself. A definition that is gone was adopted by nobody. +func AdoptedElsewhere(ctx context.Context, st Store, rdName string) (bool, error) { + current, err := st.ResourceDefinitions().GetUncached(ctx, rdName) + if errors.Is(err, ErrNotFound) { + return false, nil + } + + if err != nil { + return false, fmt.Errorf("read %q back to check for a request that adopted it: %w", rdName, err) + } + + return current.Props[RestoreAdoptedProp] != "", nil +} + +// HoldRollback is the creating half of the handshake ClaimAdoptedLeftover +// describes, for a door that rolls back synchronously: it writes the +// in-progress mark through the API server, then reads the definition back +// past the cache. Nil means the rollback may delete. ErrRollbackYielded means a +// request adopted the definition, ErrRollbackAnswered that it already reads as +// finished (ReadsAsFinished, with needReplica the door's), and any other error +// means the handshake could not be held; all of them mean nothing may be +// deleted, and the mark is taken back off so a retry is not refused over a +// definition left whole. +// +// A definition already gone answers ErrRollbackGone: nothing of it is left to +// remove, and deleting by name would take whatever stands under the name by +// then. +func HoldRollback(ctx context.Context, st Store, rdName string, needReplica bool) error { + err := setRollbackMark(ctx, st, rdName, RollbackInProgress) + if errors.Is(err, ErrNotFound) { + return fmt.Errorf("%q: %w", rdName, ErrRollbackGone) + } + + if err != nil { + return standDown(ctx, st, rdName, fmt.Errorf("mark %q as rolling back: %w", rdName, err)) + } + + adopted, err := AdoptedElsewhere(ctx, st, rdName) + if err != nil { + return standDown(ctx, st, rdName, err) + } + + if adopted { + return standDown(ctx, st, rdName, fmt.Errorf("%q: %w", rdName, ErrRollbackYielded)) + } + + finished, err := ReadsAsFinished(ctx, st, rdName, needReplica) + if err != nil { + return standDown(ctx, st, rdName, err) + } + + if finished { + return standDown(ctx, st, rdName, fmt.Errorf("%q: %w", rdName, ErrRollbackAnswered)) + } + + return nil +} + +// ErrRollbackGone reports a rollback whose definition was already gone, so +// there was nothing to remove. +var ErrRollbackGone = errors.New("the definition was already gone") + +// errRollbackMarkNotOurs is a definition that no longer carries the +// in-progress mark the rollback wrote: an operator cleared it, or the +// definition under the name is not the one the mark went on. +var errRollbackMarkNotOurs = errors.New("the rollback mark is no longer on the definition") + +// EnterDestructiveRollback replaces the in-progress mark HoldRollback wrote +// with step, the first step that takes the definition apart, before that step +// runs. In-progress is then only ever over a definition left whole, which is +// what lets an operator clear it (RollbackMarkClearRefusal). The write checks +// the definition still carries the in-progress mark, the identity check a +// delete by name needs: a definition recreated under the name since carries +// none. +// +// ErrRollbackGone means nothing is left to remove. Any other error means +// nothing may be deleted; the in-progress mark is taken back off unless it was +// not this rollback's to begin with. +func EnterDestructiveRollback(ctx context.Context, st Store, rdName, step string) error { + err := st.ResourceDefinitions().PatchResourceDefinitionSpec(ctx, rdName, + func(rd *apiv1.ResourceDefinition) error { + if rd.Props[RollbackAbandonedProp] != RollbackInProgress { + return errRollbackMarkNotOurs + } + + rd.Props[RollbackAbandonedProp] = step + + return nil + }) + + switch { + case err == nil: + return nil + case errors.Is(err, ErrNotFound): + return fmt.Errorf("%q: %w", rdName, ErrRollbackGone) + case errors.Is(err, errRollbackMarkNotOurs): + return fmt.Errorf("%q: %w", rdName, err) + default: + return standDown(ctx, st, rdName, fmt.Errorf("mark %q as being taken apart: %w", rdName, err)) + } +} + +// standDown takes the in-progress mark back off a definition a rollback leaves +// whole, for why. A mark that cannot be taken off refuses every retry over the +// definition, so that is said beside why rather than dropped. +func standDown(ctx context.Context, st Store, rdName string, why error) error { + err := ReleaseRollback(ctx, st, rdName) + if err != nil { + return errors.Join(why, fmt.Errorf("%w; a retry is refused until the mark is cleared", err)) + } + + return why +} + +// ReleaseRollback takes the in-progress mark HoldRollback put back off, for a +// rollback that held the handshake and then could not delete: the definition is +// still there and nobody is rolling it back any more, so a retry has to be able +// to resume it rather than meet a rollback that will never finish. +func ReleaseRollback(ctx context.Context, st Store, rdName string) error { + err := setRollbackMark(ctx, st, rdName, "") + if err != nil && !errors.Is(err, ErrNotFound) { + return fmt.Errorf("clear the rollback mark on %q: %w", rdName, err) + } + + return nil +} + +// setRollbackMark writes the rollback mark, or removes it when step is empty. +func setRollbackMark(ctx context.Context, st Store, rdName, step string) error { + return st.ResourceDefinitions().PatchResourceDefinitionSpec(ctx, rdName, //nolint:wrapcheck // callers wrap with the definition they name + func(rd *apiv1.ResourceDefinition) error { + if step == "" { + delete(rd.Props, RollbackAbandonedProp) + + return nil + } + + if rd.Props == nil { + rd.Props = map[string]string{} + } + + rd.Props[RollbackAbandonedProp] = step + + return nil + }) +} diff --git a/pkg/store/restore_target_test.go b/pkg/store/restore_target_test.go new file mode 100644 index 00000000..c39c9559 --- /dev/null +++ b/pkg/store/restore_target_test.go @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: Apache-2.0 + +package store_test + +import ( + "context" + "errors" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// cacheBehind answers the cached volume and replica listings from a cache +// that has not caught up: it holds no volumes and no replicas. The uncached +// listings read the backend, the way the API server does. +type cacheBehind struct{ store.Store } + +type cacheBehindVolumes struct{ store.VolumeDefinitionStore } + +func (c cacheBehindVolumes) List(context.Context, string) ([]apiv1.VolumeDefinition, error) { + return nil, nil +} + +func (c cacheBehindVolumes) ListUncached(ctx context.Context, rdName string) ([]apiv1.VolumeDefinition, error) { + return c.VolumeDefinitionStore.List(ctx, rdName) //nolint:wrapcheck // pass-through test double +} + +type cacheBehindReplicas struct{ store.ResourceStore } + +func (c cacheBehindReplicas) ListByDefinition(context.Context, string) ([]apiv1.Resource, error) { + return nil, nil +} + +func (c cacheBehindReplicas) ListByDefinitionUncached(ctx context.Context, rdName string) ([]apiv1.Resource, error) { + return c.ResourceStore.ListByDefinition(ctx, rdName) //nolint:wrapcheck // pass-through test double +} + +func (c cacheBehind) VolumeDefinitions() store.VolumeDefinitionStore { + return cacheBehindVolumes{c.Store.VolumeDefinitions()} +} + +func (c cacheBehind) Resources() store.ResourceStore { + return cacheBehindReplicas{c.Store.Resources()} +} + +func seedPreparedTarget(t *testing.T, st store.Store) *apiv1.Snapshot { + t.Helper() + + ctx := t.Context() + + for _, name := range []string{"pvc-src", "pvc-dst"} { + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: name}); err != nil { + t.Fatalf("seed %s: %v", name, err) + } + } + + if err := st.VolumeDefinitions().Create(ctx, "pvc-dst", + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 1 << 20}); err != nil { + t.Fatalf("seed the restored volume: %v", err) + } + + return &apiv1.Snapshot{ + Name: "snap", ResourceName: "pvc-src", + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 1 << 20}}, + } +} + +// linstor-csi's volume restore and the resource restore after it can land on +// different replicas of the server. A cache on the second that had not seen +// the first's volumes refused a target that was prepared. +func TestPreparedRestoreTargetReadsTheVolumesPastTheCache(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + snap := seedPreparedTarget(t, backend) + + rd, err := backend.ResourceDefinitions().Get(t.Context(), "pvc-dst") + if err != nil { + t.Fatalf("read the target: %v", err) + } + + prepared, err := store.PreparedRestoreTarget(t.Context(), cacheBehind{backend}, &rd, snap) + if err != nil || !prepared { + t.Errorf("prepared = %v (%v), want a prepared target", prepared, err) + } +} + +// The same lag on the replicas took a target that holds a live replica, which +// may hold somebody's data. +func TestPreparedRestoreTargetReadsTheReplicasPastTheCache(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + snap := seedPreparedTarget(t, backend) + + if err := backend.Resources().Create(t.Context(), + &apiv1.Resource{Name: "pvc-dst", NodeName: "node-a"}); err != nil { + t.Fatalf("seed the live replica: %v", err) + } + + rd, err := backend.ResourceDefinitions().Get(t.Context(), "pvc-dst") + if err != nil { + t.Fatalf("read the target: %v", err) + } + + prepared, err := store.PreparedRestoreTarget(t.Context(), cacheBehind{backend}, &rd, snap) + if prepared { + t.Errorf("a target with a live replica the cache had not seen was taken (%v)", err) + } +} + +// The definition the caller read can be the cache's too; one that has been +// marked since is no longer prepared. +func TestPreparedRestoreTargetReadsTheDefinitionPastTheCache(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + snap := seedPreparedTarget(t, backend) + + stale, err := backend.ResourceDefinitions().Get(t.Context(), "pvc-dst") + if err != nil { + t.Fatalf("read the target: %v", err) + } + + if _, err := store.AdoptPreparedRestoreTarget(t.Context(), backend, "pvc-dst", &apiv1.Snapshot{ + Name: "other", ResourceName: "pvc-src", + }); err != nil { + t.Fatalf("mark the target for another restore: %v", err) + } + + prepared, err := store.PreparedRestoreTarget(t.Context(), backend, &stale, snap) + if prepared || errors.Is(err, store.ErrRestoreTargetLayers) { + t.Errorf("a target marked since the caller read it was taken (%v)", err) + } +} + +// A target another request for the same restore already marked is that +// request's: the marker stays, even over the replicas that request placed. +func TestAdoptPreparedRestoreTargetLeavesAMarkerItDidNotWrite(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + snap := seedPreparedTarget(t, backend) + + if _, err := store.AdoptPreparedRestoreTarget(ctx, backend, "pvc-dst", snap); err != nil { + t.Fatalf("mark the target as the first request did: %v", err) + } + + if err := backend.Resources().Create(ctx, &apiv1.Resource{Name: "pvc-dst", NodeName: "node-a"}); err != nil { + t.Fatalf("place the first request's replica: %v", err) + } + + adopted, err := store.AdoptPreparedRestoreTarget(ctx, backend, "pvc-dst", snap) + if adopted || err != nil { + t.Errorf("adopt over a marker already there = (%v, %v), want (false, nil)", adopted, err) + } + + rd, err := backend.ResourceDefinitions().Get(ctx, "pvc-dst") + if err != nil { + t.Fatalf("read the target: %v", err) + } + + if rd.Props[store.RestoreFromSnapshotProp] == "" { + t.Error("the marker the first request wrote was taken off") + } +} + +// Two restores of different snapshots into one prepared target: the second +// adoption must not write its marker over the first, or every replica placed +// from then on restores from a snapshot the first restore was not told about. +func TestAdoptPreparedRestoreTargetRefusesATargetAnotherRestoreMarked(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + snap := seedPreparedTarget(t, backend) + + if _, err := store.AdoptPreparedRestoreTarget(ctx, backend, "pvc-dst", snap); err != nil { + t.Fatalf("mark the target for the first restore: %v", err) + } + + other := &apiv1.Snapshot{Name: "other", ResourceName: snap.ResourceName} + + adopted, err := store.AdoptPreparedRestoreTarget(ctx, backend, "pvc-dst", other) + if adopted || !errors.Is(err, store.ErrRestoreTargetTaken) { + t.Errorf("adopt for another snapshot = (%v, %v), want ErrRestoreTargetTaken", adopted, err) + } + + rd, err := backend.ResourceDefinitions().Get(ctx, "pvc-dst") + if err != nil { + t.Fatalf("read the target: %v", err) + } + + if want := snap.ResourceName + ":" + snap.Name; rd.Props[store.RestoreFromSnapshotProp] != want { + t.Errorf("marker = %q, want the first restore's %q", rd.Props[store.RestoreFromSnapshotProp], want) + } +} + +// This restore's own leftover is judged the way the REST door judges it: +// unfinished without a live replica, finished with one, refused while every +// replica is going away, over a group that is gone, or over a rollback that +// gave up, and not this restore's to judge under another marker or none. +func TestJudgeRestoreLeftover(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + props map[string]string + group string + replicas []apiv1.Resource + own bool + want store.LeftoverProgress + wantErr error + }{ + {name: "marked-no-replicas", props: marked(), own: true, want: store.LeftoverUnfinished}, + {name: "marked-one-live", props: marked(), own: true, want: store.LeftoverFinished, replicas: []apiv1.Resource{ + {NodeName: "node-a"}, + {NodeName: "node-b", Flags: []string{apiv1.ResourceFlagDelete}}, + }}, + { + name: "marked-all-deleting", props: marked(), own: true, want: store.LeftoverTearingDown, + wantErr: store.ErrRestoreTargetTearingDown, + replicas: []apiv1.Resource{{NodeName: "node-a", Flags: []string{apiv1.ResourceFlagDelete}}}, + }, + { + name: "marked-group-gone", props: marked(), group: "grp-gone", own: true, + want: store.LeftoverUnfinished, wantErr: store.ErrRestoreTargetGroupGone, + }, + { + name: "marked-rollback-gave-up", own: true, want: store.LeftoverUnfinished, + props: map[string]string{ + store.RestoreFromSnapshotProp: "pvc-src:snap", + store.RollbackAbandonedProp: "replicas", + }, + wantErr: store.ErrRestoreRollbackAbandoned, + }, + {name: "another-restore", props: map[string]string{store.RestoreFromSnapshotProp: "pvc-src:other"}}, + {name: "unmarked"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + snap := seedPreparedTarget(t, backend) + + if tc.props != nil || tc.group != "" { + if err := backend.ResourceDefinitions().PatchResourceDefinitionSpec(ctx, "pvc-dst", + func(rd *apiv1.ResourceDefinition) error { + rd.Props = tc.props + rd.ResourceGroupName = tc.group + + return nil + }); err != nil { + t.Fatalf("mark the target: %v", err) + } + } + + for i := range tc.replicas { + replica := tc.replicas[i] + replica.Name = "pvc-dst" + + if err := backend.Resources().Create(ctx, &replica); err != nil { + t.Fatalf("seed replica: %v", err) + } + } + + got, own, err := store.JudgeRestoreLeftover(ctx, backend, "pvc-dst", snap) + if own != tc.own || (own && got != tc.want) || !errors.Is(err, tc.wantErr) { + t.Errorf("judged = (%v, %v, %v), want (%v, %v, %v)", got, own, err, tc.want, tc.own, tc.wantErr) + } + }) + } +} + +func marked() map[string]string { + return map[string]string{store.RestoreFromSnapshotProp: "pvc-src:snap"} +} + +// Every marker carries the record of what the snapshot held, the prepared +// door's included, so a leftover can be judged once the snapshot is gone. +func TestAdoptPreparedRestoreTargetRecordsTheSnapshotVolumes(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + snap := seedPreparedTarget(t, backend) + + if _, err := store.AdoptPreparedRestoreTarget(ctx, backend, "pvc-dst", snap); err != nil { + t.Fatalf("adopt: %v", err) + } + + rd, err := backend.ResourceDefinitions().Get(ctx, "pvc-dst") + if err != nil { + t.Fatalf("read the target: %v", err) + } + + if got, want := rd.Props[store.RestoreVolumesProp], store.EncodeRestoreVolumes(snap.VolumeDefinitions); got != want { + t.Errorf("record = %q, want %q", got, want) + } +} + +// The record describes THIS definition's restore. Copied onto a snapshot taken +// of it, and from there onto a definition restored from that snapshot, it would +// judge the second definition against the first one's volumes. +func TestTravellingPropsDropsTheRestoreRecord(t *testing.T) { + t.Parallel() + + out := store.TravellingProps(map[string]string{ + store.RestoreVolumesProp: "0=1024", + "Aux/keep": "yes", + }) + + if _, ok := out[store.RestoreVolumesProp]; ok { + t.Error("the restore record travelled") + } + + if out["Aux/keep"] != "yes" { + t.Error("an ordinary prop did not travel") + } +} + +// A record that does not parse is no record, never a shape to judge against. +func TestRecordedRestoreVolumes(t *testing.T) { + t.Parallel() + + want := []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 1024}, {VolumeNumber: 3, SizeKib: 2048}} + + got, ok := store.RecordedRestoreVolumes(map[string]string{ + store.RestoreVolumesProp: store.EncodeRestoreVolumes(want), + }) + if !ok || len(got) != 2 || got[1].VolumeNumber != 3 || got[1].SizeKib != 2048 { + t.Errorf("round trip = (%v, %v), want %v", got, ok, want) + } + + for _, bad := range []string{"0", "x=1", "0=y"} { + if _, ok := store.RecordedRestoreVolumes(map[string]string{store.RestoreVolumesProp: bad}); ok { + t.Errorf("%q parsed as a record", bad) + } + } + + if _, ok := store.RecordedRestoreVolumes(nil); ok { + t.Error("no prop parsed as a record") + } +} diff --git a/pkg/store/rollback_hold_test.go b/pkg/store/rollback_hold_test.go new file mode 100644 index 00000000..8cebd360 --- /dev/null +++ b/pkg/store/rollback_hold_test.go @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: Apache-2.0 + +package store_test + +import ( + "context" + "errors" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +var errPatchRefused = errors.New("apiserver refused the patch") + +// patchRefusingRDs fails every spec patch, the way an API server that cannot be +// reached does. +type patchRefusingRDs struct{ store.ResourceDefinitionStore } + +func (patchRefusingRDs) PatchResourceDefinitionSpec( + context.Context, string, func(*apiv1.ResourceDefinition) error, +) error { + return errPatchRefused +} + +type patchRefusingStore struct{ store.Store } + +func (p patchRefusingStore) ResourceDefinitions() store.ResourceDefinitionStore { + return patchRefusingRDs{p.Store.ResourceDefinitions()} +} + +var errReadRefused = errors.New("apiserver refused the read") + +// readRefusingRDs lets the mark through and fails the authoritative read that +// follows it, the read the rollback decides on. +type readRefusingRDs struct{ store.ResourceDefinitionStore } + +func (readRefusingRDs) GetUncached(context.Context, string) (apiv1.ResourceDefinition, error) { + return apiv1.ResourceDefinition{}, errReadRefused +} + +type readRefusingStore struct{ store.Store } + +func (r readRefusingStore) ResourceDefinitions() store.ResourceDefinitionStore { + return readRefusingRDs{r.Store.ResourceDefinitions()} +} + +// The rollback half of the handshake: a definition no request adopted may be +// deleted, one a request adopted is left to it, and one whose mark could not be +// written is not deleted at all. +func TestHoldRollback(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + adopted bool + finished bool + refuse bool + refuseRead bool + want error + mayGo bool + }{ + {name: "nobody adopted it", mayGo: true}, + {name: "a request adopted it", adopted: true, want: store.ErrRollbackYielded}, + {name: "it already reads as finished", finished: true, want: store.ErrRollbackAnswered}, + {name: "the mark cannot be written", refuse: true, want: errPatchRefused}, + {name: "the adoption cannot be read", refuseRead: true, want: errReadRefused}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + + props := map[string]string{} + if tc.adopted { + props[store.RestoreAdoptedProp] = "2026-10-07T00:00:00Z" + } + + if err := backend.ResourceDefinitions().Create(ctx, + &apiv1.ResourceDefinition{Name: "pvc-new", Props: props}); err != nil { + t.Fatalf("seed: %v", err) + } + + if tc.finished { + seedFinishedLeftover(t, backend, "pvc-new") + } + + var st store.Store = backend + if tc.refuse { + st = patchRefusingStore{backend} + } + + if tc.refuseRead { + st = readRefusingStore{backend} + } + + err := store.HoldRollback(ctx, st, "pvc-new", true) + if tc.mayGo { + if err != nil { + t.Fatalf("HoldRollback = %v, want nil", err) + } + + return + } + + if !errors.Is(err, tc.want) { + t.Fatalf("HoldRollback = %v, want %v", err, tc.want) + } + + if !errors.Is(tc.want, store.ErrRollbackAnswered) && errors.Is(err, store.ErrRollbackAnswered) { + t.Fatalf("HoldRollback = %v: a failure reported as a finished leftover", err) + } + + if !errors.Is(tc.want, store.ErrRollbackYielded) && errors.Is(err, store.ErrRollbackYielded) { + t.Fatalf("HoldRollback = %v: a failure reported as a yield", err) + } + + rd, getErr := backend.ResourceDefinitions().Get(ctx, "pvc-new") + if getErr != nil { + t.Fatalf("read back: %v", getErr) + } + + if mark, ok := rd.Props[store.RollbackAbandonedProp]; ok { + t.Errorf("a rollback that may not delete left mark %q", mark) + } + }) + } +} + +// seedFinishedLeftover gives a definition a volume and a live replica, which +// is what a replay and the clone status poll answer as finished. +func seedFinishedLeftover(t *testing.T, st store.Store, rdName string) { + t.Helper() + + ctx := t.Context() + + if err := st.VolumeDefinitions().Create(ctx, rdName, + &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 1024}); err != nil { + t.Fatalf("seed the volume: %v", err) + } + + if err := st.Resources().Create(ctx, &apiv1.Resource{Name: rdName, NodeName: "node-a"}); err != nil { + t.Fatalf("seed the replica: %v", err) + } +} + +// A definition being deleted is not finished, to the replay, the poll or the +// rollback: it holds every volume and a replica and is still not spared. +func TestReadsAsFinishedIsFalseForADefinitionBeingDeleted(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + deleting bool + want bool + }{ + {name: "whole", want: true}, + {name: "being-deleted", deleting: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + + rd := &apiv1.ResourceDefinition{Name: "pvc-flagged"} + if tc.deleting { + rd.Flags = []string{apiv1.ResourceFlagDelete} + } + + if err := backend.ResourceDefinitions().Create(ctx, rd); err != nil { + t.Fatalf("seed: %v", err) + } + + seedFinishedLeftover(t, backend, "pvc-flagged") + + got, err := store.ReadsAsFinished(ctx, backend, "pvc-flagged", true) + if err != nil || got != tc.want { + t.Errorf("ReadsAsFinished = %v, %v; want %v", got, err, tc.want) + } + + err = store.HoldRollback(ctx, backend, "pvc-flagged", true) + if tc.want != errors.Is(err, store.ErrRollbackAnswered) { + t.Errorf("HoldRollback = %v, want answered=%v", err, tc.want) + } + }) + } +} + +// The rule an operator's edit of a definition is held to over the rollback +// mark: the mark over a definition left whole may be deleted, one naming a +// step past that may not, and an edit that leaves the mark alone goes through +// whatever it says. +func TestRollbackMarkClearRefusal(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + before, after map[string]string + refused bool + }{ + {name: "no-mark", before: map[string]string{"Aux/x": "1"}, after: map[string]string{}}, + { + name: "in-progress-deleted", + before: map[string]string{store.RollbackAbandonedProp: store.RollbackInProgress}, after: map[string]string{}, + }, + { + name: "snapshots-deleted", + before: map[string]string{store.RollbackAbandonedProp: store.RollbackStepSnapshots}, after: map[string]string{}, + }, + { + name: "read-snapshots-deleted", + before: map[string]string{store.RollbackAbandonedProp: store.RollbackStepReadSnapshots}, after: map[string]string{}, + }, + { + name: "step-deleted", + before: map[string]string{store.RollbackAbandonedProp: "reap-replicas"}, after: map[string]string{}, + refused: true, + }, + { + name: "step-kept-other-set", + before: map[string]string{store.RollbackAbandonedProp: "reap-replicas"}, + after: map[string]string{store.RollbackAbandonedProp: "reap-replicas", "Aux/x": "1"}, + }, + { + name: "step-kept-other-deleted", + before: map[string]string{store.RollbackAbandonedProp: "reap-replicas", "Aux/x": "1"}, + after: map[string]string{store.RollbackAbandonedProp: "reap-replicas"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + err := store.RollbackMarkClearRefusal(tc.before, tc.after) + if tc.refused != errors.Is(err, store.ErrRollbackStepMarkKept) { + t.Errorf("RollbackMarkClearRefusal = %v, want refused=%v", err, tc.refused) + } + }) + } +} + +// secondPatchRefusingRDs lets the in-progress mark through and refuses the +// patch that would take it back off. +type secondPatchRefusingRDs struct { + store.ResourceDefinitionStore + + patches *int +} + +func (r secondPatchRefusingRDs) PatchResourceDefinitionSpec( + ctx context.Context, name string, mutate func(*apiv1.ResourceDefinition) error, +) error { + *r.patches++ + if *r.patches > 1 { + return errPatchRefused + } + + return r.ResourceDefinitionStore.PatchResourceDefinitionSpec(ctx, name, mutate) //nolint:wrapcheck // test double +} + +type secondPatchRefusingStore struct { + store.Store + + patches *int +} + +func (s secondPatchRefusingStore) ResourceDefinitions() store.ResourceDefinitionStore { + return secondPatchRefusingRDs{ResourceDefinitionStore: s.Store.ResourceDefinitions(), patches: s.patches} +} + +// A rollback that yields and cannot take its mark back off leaves a mark every +// retry is refused over, and says so rather than dropping the failure. +func TestHoldRollbackReportsAMarkItCouldNotTakeOff(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + if err := backend.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{ + Name: "pvc-stuck", Props: map[string]string{store.RestoreAdoptedProp: "2026-01-01T00:00:00Z"}, + }); err != nil { + t.Fatalf("seed the adopted definition: %v", err) + } + + patches := 0 + + err := store.HoldRollback(t.Context(), secondPatchRefusingStore{Store: backend, patches: &patches}, "pvc-stuck", true) + if !errors.Is(err, store.ErrRollbackYielded) { + t.Fatalf("hold over an adopted definition = %v, want ErrRollbackYielded", err) + } + + if !errors.Is(err, errPatchRefused) { + t.Errorf("hold whose mark could not be taken off = %v, want the refused patch said too", err) + } +} diff --git a/pkg/store/rollback_identity_test.go b/pkg/store/rollback_identity_test.go new file mode 100644 index 00000000..fa035830 --- /dev/null +++ b/pkg/store/rollback_identity_test.go @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: Apache-2.0 + +package store_test + +import ( + "errors" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// A rollback whose definition is already gone has nothing of it to remove, +// and says so, so the door does not go on to delete whatever stands under the +// name by then. +func TestHoldRollbackReportsADefinitionAlreadyGone(t *testing.T) { + t.Parallel() + + err := store.HoldRollback(t.Context(), store.NewInMemory(), "pvc-gone", true) + if !errors.Is(err, store.ErrRollbackGone) { + t.Errorf("HoldRollback over a missing definition = %v, want ErrRollbackGone", err) + } +} + +// The step mark goes on only over the rollback's own in-progress mark: a +// definition that no longer carries it is not the one the rollback holds. +func TestEnterDestructiveRollback(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + mark string + missing bool + want error + wantMark string + }{ + {name: "its own mark", mark: store.RollbackInProgress, wantMark: store.RollbackStepDeleteDefinition}, + {name: "the mark was cleared", mark: "", wantMark: ""}, + {name: "the definition is gone", missing: true, want: store.ErrRollbackGone}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + + if !tc.missing { + props := map[string]string{} + if tc.mark != "" { + props[store.RollbackAbandonedProp] = tc.mark + } + + if err := backend.ResourceDefinitions().Create(ctx, + &apiv1.ResourceDefinition{Name: "pvc-new", Props: props}); err != nil { + t.Fatalf("seed: %v", err) + } + } + + err := store.EnterDestructiveRollback(ctx, backend, "pvc-new", store.RollbackStepDeleteDefinition) + + switch { + case tc.want != nil: + if !errors.Is(err, tc.want) { + t.Fatalf("EnterDestructiveRollback = %v, want %v", err, tc.want) + } + + return + case tc.mark == store.RollbackInProgress: + if err != nil { + t.Fatalf("EnterDestructiveRollback = %v, want nil", err) + } + default: + if err == nil { + t.Fatal("EnterDestructiveRollback went ahead over a mark that is not its own") + } + } + + rd, err := backend.ResourceDefinitions().Get(ctx, "pvc-new") + if err != nil { + t.Fatalf("read back: %v", err) + } + + if got := rd.Props[store.RollbackAbandonedProp]; got != tc.wantMark { + t.Errorf("rollback mark = %q, want %q", got, tc.wantMark) + } + }) + } +} diff --git a/stand/blockstor-apiserver-deploy.yaml b/stand/blockstor-apiserver-deploy.yaml index 75a05c92..c70d1b98 100644 --- a/stand/blockstor-apiserver-deploy.yaml +++ b/stand/blockstor-apiserver-deploy.yaml @@ -77,7 +77,7 @@ spec: # The REST server's graceful-shutdown window is derived from the # detached rollback budget (pkg/rest/rg_deleted_race.go); this has to # outlive it, or a SIGTERM kills a compensation mid-cascade. - terminationGracePeriodSeconds: 20 + terminationGracePeriodSeconds: 21 # Spread replicas across nodes when possible so a single # node loss doesn't take all apiserver pods at once. topologySpreadConstraints: diff --git a/stand/blockstor-deploy.yaml b/stand/blockstor-deploy.yaml index 6c7472c5..7b762e87 100644 --- a/stand/blockstor-deploy.yaml +++ b/stand/blockstor-deploy.yaml @@ -74,7 +74,7 @@ spec: # The REST server's graceful-shutdown window is derived from the # detached rollback budget (pkg/rest/rg_deleted_race.go); this has to # outlive it, or a SIGTERM kills a compensation mid-cascade. - terminationGracePeriodSeconds: 20 + terminationGracePeriodSeconds: 21 containers: - name: manager image: __REGISTRY__/blockstor-controller:dev From 877168269fa148aa06ab47ecac0027fc8548c409 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 7 Oct 2026 10:00:27 +0200 Subject: [PATCH 3/4] test(e2e): pin the clone retry semantics on a stand and in the replay The unit tests run the handlers against the in-memory store. The cli-matrix cell drives the same retries against a live stand: a clone replayed after the source was resized, leftover shapes, and the delete_namespaces field over raw REST, since `rd clone` has no flag for it and it only rides on golinstor's props-modify envelope. The replay workflow records the CLI's view of the same sequence. The workflow waits on state through the runner's awaits rather than a CLI list: for about two seconds after a create the python client indexes a replica's DRBD layer before the payload is filled and dies with KeyError, which is not this workflow's subject. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- tests/e2e/cli-matrix/README.md | 1 + .../cli-matrix/rd-clone-retry-semantics.sh | 231 ++++++++++++++++++ .../replay/rd-clone-retry-semantics.yaml | 104 ++++++++ 3 files changed, 336 insertions(+) create mode 100755 tests/e2e/cli-matrix/rd-clone-retry-semantics.sh create mode 100644 tests/operator-harness/replay/rd-clone-retry-semantics.yaml diff --git a/tests/e2e/cli-matrix/README.md b/tests/e2e/cli-matrix/README.md index b55fb856..6bd6dbff 100644 --- a/tests/e2e/cli-matrix/README.md +++ b/tests/e2e/cli-matrix/README.md @@ -42,6 +42,7 @@ A cell counts as **FAIL** if either leg times out, the `linstor` CLI exits non-z | `r-l-conns-shapes.sh` | 331 | Conns/State column contract: parses `linstor r l` JSON across (Healthy, Disconnected peer, Diskless, TieBreaker) shapes and pins observer's events2 translation. | | `snap-restore-snapshotless-node-rejected.sh` | 397 | P0 DATA INTEGRITY. `snapshot resource restore` onto a node NOT holding the snapshot is rejected (no silent empty replica, no orphan RD); restoring onto the snapshot's own nodes converges UpToDate AND every replica holds the real snapshot bytes (marker read per-replica), never a silently-empty UpToDate copy. | | `rd-clone-vd-data-plane.sh` | 020 | `linstor rd clone ` on a VD-bearing source (plain CLI body, no `use_zfs_clone`) AND the raw-REST `use_zfs_clone=true` body linstor-csi sends both materialise a real clone: 2 replicas UpToDate, marker bytes from the source present on EVERY clone replica (promote each in turn), clone status COMPLETE, internal `clone-` snapshot visible on the source. Pre-fix: 400 on `use_zfs_clone`, 501 on VD-bearing sources (Bug 114 gate). | +| `rd-clone-retry-semantics.sh` | parity | The clone endpoint's retry contract. `linstor rd clone ` run twice answers the same way twice, and still does after `vd set-size` grows the source: a finished clone is a copy of a point-in-time and owes the source nothing, while a guard comparing the internal snapshot to the live source made an ordinary resize refuse every later retry permanently (the snapshot outlives the clone by design, so the refusal's own correction cannot be followed). Also pins a clone body carrying `delete_namespaces`, driven over raw REST because the verb has no flag for it — it is a wire field from golinstor's `GenericPropsModify`, and a 400 before the triple was declared. | | `encryption-passphrase-luks-rd.sh` | 023 | Secret-only LUKS flow: `linstor encryption create-passphrase` alone (legacy `DrbdOptions/EncryptPassphrase` controller prop asserted ABSENT throughout) unlocks `rd c -l drbd,luks,storage` + autoplace to UpToDate, and the Secret-backed passphrase actually opens the LUKS header on each replica's backing device. Requires the Bug-023 fix (PR #143); pre-fix the rd-create is rejected with "LUKS layer requires DrbdOptions/EncryptPassphrase to be set first". | | `vd-modify-preserves-drbd-minor.sh` | 433 | A legal in-bounds VD-scoped modify (`vd set-size` grow / `vd set-property`) must NOT change the per-volume DRBDMinor — the /dev/drbd device identity. Creates rd-a (lowest minor Ma) + rd-b (next minor Mb), frees Ma by deleting rd-a, then modifies rd-b: its DRBDMinor AND device path must stay Mb over a settle window. Pre-fix `wireToCRDVD` dropped the minor on the VD-scoped write-back and the allocator re-stamped the freed lower Ma — a permanent device-identity change on a live resized volume. | | `rg-modify-invalid-layer-rejected.sh` | 434 | `rg modify --layer-list storage,drbd` (STORAGE-before-DRBD, the ordering the create path refuses) must be rejected, the RG must keep its valid `[DRBD,STORAGE]` stack, and an RD inheriting from the RG must get the valid stack — never `[STORAGE,DRBD]`. Pre-fix `rg modify` gated only place_count and `handleRDCreate` validated the explicit layer_list BEFORE the RG inherit, so an invalid stack reached a persisted RD via the modify→inherit chain. | diff --git a/tests/e2e/cli-matrix/rd-clone-retry-semantics.sh b/tests/e2e/cli-matrix/rd-clone-retry-semantics.sh new file mode 100755 index 00000000..271a45e7 --- /dev/null +++ b/tests/e2e/cli-matrix/rd-clone-retry-semantics.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env bash +# +# usage: rd-clone-retry-semantics.sh WORK_DIR +# +# L6 cli-matrix cell — the clone endpoint's retry contract, and the +# props-modify triple it accepts. +# +# Three behaviours that are new on this surface and that unit tests +# cannot reach, because each of them is about what the OPERATOR sees +# on a second invocation of the same verb: +# +# A. Replaying a finished clone succeeds. linstor-csi re-issues +# CreateVolume whenever a response is lost or external-provisioner +# restarts, so `rd clone ` run twice must answer the +# same way twice. A finished clone is a copy of a point-in-time. +# +# B. Replaying it AFTER the source grows still succeeds. This is the +# regression the guard introduced: comparing the internal snapshot +# against the live source made an ordinary `vd set-size` poison +# every later retry, permanently — the snapshot is deterministic +# and outlives the clone by design (delta row 82), so no repeat +# could ever get past it, and its own correction ("delete the +# snapshot") cannot be followed, because that snapshot is the +# origin the existing clone depends on. +# +# C. A clone body carrying `delete_namespaces` is accepted. The verb +# has no flag for it, so this is driven over raw REST: the field +# reaches the endpoint from golinstor's GenericPropsModify. The +# body is decoded with DisallowUnknownFields, so before the triple +# was declared an undeclared field was a 400 before any of the +# handler ran. +# +# Contract: +# 1. first clone: exit 0, target materialises replicas, UpToDate. +# 2. second clone, same argv: exit 0, and the target still holds +# exactly the volume the first one produced (a replay must not +# re-shape what it replays). +# 3. `vd set-size` on the source, then a third clone: exit 0. +# 4. clone POST carrying delete_namespaces: HTTP 201. +# +# Pool: `stand` (FILE_THIN), the pool the sibling clone cell uses. +# +# Unit pins: pkg/rest/rd_clone_leftover_shape_test.go (replay after +# resize, leftover shapes) and pkg/rest/props_delete_namespaces_test.go +# (delete_namespaces). This cell is the stand-side companion. + +set -euo pipefail + +WORK_DIR=${1:?work_dir required} +export KUBECONFIG="$WORK_DIR/kubeconfig" + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" + +require_workers 2 + +linstor_cli_setup + +SRC=cli-matrix-retry-src +DST=cli-matrix-retry-dst +DST_NS=cli-matrix-retry-ns +DST_CTL=cli-matrix-retry-ctl +POOL=${POOL:-stand} + +N1=$WORKER_1 +N2=$WORKER_2 + +# The strict cleanup drains the cluster, the orphan check then looks for the +# kernel and storage residue it does not, through the port-forward, which goes +# last. A passing cell either of them finds dirty is failed here, so the next +# cell is not blamed for it. errexit goes off once the cell's status is read: +# under set -e a failed cell would end the trap before any of it ran. +cleanup() { + local rc=$? clean=0 rd + set +e + strict_cleanup_on_exit "${_STRICT_CLEANUP_RDS[@]}" || clean=1 + for rd in "${_STRICT_CLEANUP_RDS[@]}"; do + assert_no_orphans "$rd" || clean=1 + done + linstor_cli_teardown + if [[ $rc -eq 0 && $clean -ne 0 ]]; then + echo "FAIL: the cluster could not be cleaned after the cell; PASS demoted to FAIL" >&2 + rc=1 + fi + exit "$rc" +} +register_strict_cleanup "$DST" "$DST_NS" "$DST_CTL" "$SRC" +trap cleanup EXIT + +# clone_volume_sizes — every volume of the target as `=`, +# sorted, as the controller reports them: a replay that added or dropped a +# volume changes it as surely as one that resized one. +clone_volume_sizes() { + local rd=$1 + "${LCTL[@]}" --machine-readable volume-definition list --resource-definitions "$rd" \ + 2>/dev/null | jq -r '..|.volume_definitions? // empty | .[] | "\(.volume_number)=\(.size_kib)"' \ + | sort | paste -sd, - +} + +echo ">> source RD: 2 diskful replicas on $POOL" +"${LCTL[@]}" resource-definition create "$SRC" >/dev/null +"${LCTL[@]}" volume-definition create "$SRC" 64M >/dev/null +"${LCTL[@]}" resource create "$N1" "$SRC" --storage-pool="$POOL" >/dev/null +"${LCTL[@]}" resource create "$N2" "$SRC" --storage-pool="$POOL" >/dev/null +wait_uptodate "$SRC" "$N1" "$N2" + +echo ">> [A] first clone" +if ! "${LCTL[@]}" resource-definition clone "$SRC" "$DST" >/dev/null 2>&1; then + echo "FAIL: first clone exited non-zero" >&2 + exit 1 +fi + +deadline=$(( $(date +%s) + 120 )) +dst_nodes=() +while (( $(date +%s) < deadline )); do + mapfile -t dst_nodes < <(linstor_diskful_nodes "$DST") + (( ${#dst_nodes[@]} == 2 )) && break + sleep 2 +done +if (( ${#dst_nodes[@]} != 2 )); then + echo "FAIL: clone $DST never materialised 2 diskful replicas" >&2 + exit 1 +fi +wait_uptodate "$DST" "${dst_nodes[0]}" "${dst_nodes[1]}" + +size_after_first=$(clone_volume_sizes "$DST") +# An empty read compares equal to another empty read, so the size assertions +# below would pass without ever seeing a size. Refuse to proceed on one. +if ! [[ "$size_after_first" =~ ^[0-9]+=[0-9]+ ]]; then + echo "FAIL: could not read the volumes of $DST (got '$size_after_first')" >&2 + exit 1 +fi + +echo ">> [A] replay of the finished clone answers the same way" +if ! "${LCTL[@]}" resource-definition clone "$SRC" "$DST" >/dev/null 2>&1; then + echo "FAIL: replaying a finished clone exited non-zero" >&2 + echo " CSI re-issues CreateVolume whenever a response is lost" >&2 + exit 1 +fi + +size_after_replay=$(clone_volume_sizes "$DST") +if ! [[ "$size_after_replay" =~ ^[0-9]+=[0-9]+ ]]; then + echo "FAIL: could not read the volumes of $DST after the replay (got '$size_after_replay')" >&2 + exit 1 +fi +if [[ "$size_after_first" != "$size_after_replay" ]]; then + echo "FAIL: the replay re-shaped the clone ($size_after_first -> $size_after_replay)" >&2 + exit 1 +fi + +echo ">> [B] grow the source, then replay again" +"${LCTL[@]}" volume-definition set-size "$SRC" 0 128M >/dev/null +if ! "${LCTL[@]}" resource-definition clone "$SRC" "$DST" >/dev/null 2>&1; then + echo "FAIL: replay after a source resize exited non-zero" >&2 + echo " a finished clone is a copy of a point-in-time and owes the source nothing;" >&2 + echo " the internal snapshot outlives it by design, so this refusal was permanent" >&2 + exit 1 +fi + +size_after_resize=$(clone_volume_sizes "$DST") +if ! [[ "$size_after_resize" =~ ^[0-9]+=[0-9]+ ]]; then + echo "FAIL: could not read the volumes of $DST after the source resize (got '$size_after_resize')" >&2 + exit 1 +fi +if [[ "$size_after_first" != "$size_after_resize" ]]; then + echo "FAIL: the replay re-shaped the finished clone ($size_after_first -> $size_after_resize)" >&2 + exit 1 +fi + +echo ">> [C] a clone body carrying delete_namespaces is accepted" +# Driven over raw REST, not through the verb: the clone CLI has no flag for +# this (1.31.0 offers --external-name, --use-zfs-clone, --volume-passphrase, +# --layer-list, --resource-group). delete_namespaces reaches the endpoint from +# golinstor's GenericPropsModify, which is the shape under test — and the body +# is decoded with DisallowUnknownFields, so before the triple was declared an +# undeclared field was a 400 before any of the handler ran. +# The key the step puts on the source, and the one it looks for on the clone: +# the namespace as a whole is not, since the RD reconciler stamps its own +# DrbdOptions/Resource/quorum onto every placed DRBD definition. +if ! "${LCTL[@]}" resource-definition set-property "$SRC" DrbdOptions/Net/protocol C >/dev/null 2>&1; then + echo "FAIL: could not set DrbdOptions/Net/protocol on $SRC" >&2 + exit 1 +fi +# Positive control first: a clone of the same source without +# delete_namespaces carries the key. Without it, a run where props stopped +# travelling to a clone at all would pass the check below. +ctl_out="$WORK_DIR/cli-matrix-retry-ctl.json" +http_code=$(curl -sS -m 30 -o "$ctl_out" -w '%{http_code}' \ + -X POST -H 'Content-Type: application/json' \ + -d "{\"name\":\"${DST_CTL}\",\"use_zfs_clone\":true}" \ + "http://127.0.0.1:${LCTL_PORT}/v1/resource-definitions/${SRC}/clone" \ + 2>/dev/null || echo "000") +if [[ "$http_code" != "201" ]]; then + echo "FAIL: the control clone answered HTTP $http_code, want 201" >&2 + cat "$ctl_out" >&2 2>/dev/null || true + exit 1 +fi +if ! ctl_props=$("${LCTL[@]}" -m resource-definition list-properties "$DST_CTL" 2>&1) \ + || ! grep -q 'DrbdOptions/Net/protocol' <<<"$ctl_props"; then + echo "FAIL: the control clone $DST_CTL does not carry DrbdOptions/Net/protocol" >&2 + echo "$ctl_props" >&2 + exit 1 +fi + +ns_out="$WORK_DIR/cli-matrix-retry-ns.json" +http_code=$(curl -sS -m 30 -o "$ns_out" -w '%{http_code}' \ + -X POST -H 'Content-Type: application/json' \ + -d "{\"name\":\"${DST_NS}\",\"delete_namespaces\":[\"DrbdOptions\"],\"use_zfs_clone\":true}" \ + "http://127.0.0.1:${LCTL_PORT}/v1/resource-definitions/${SRC}/clone" \ + 2>/dev/null || echo "000") +if [[ "$http_code" != "201" ]]; then + echo "FAIL: clone with delete_namespaces answered HTTP $http_code, want 201" >&2 + cat "$ns_out" >&2 2>/dev/null || true + exit 1 +fi + +# Accepted is not enough: an edit accepted and then dropped is the failure +# this step exists for, so the key the namespace held has to be gone. +if ! ns_props=$("${LCTL[@]}" -m resource-definition list-properties "$DST_NS" 2>&1); then + echo "FAIL: could not list the properties of $DST_NS" >&2 + echo "$ns_props" >&2 + exit 1 +fi +if grep -q 'DrbdOptions/Net/protocol' <<<"$ns_props"; then + echo "FAIL: the clone still carries DrbdOptions/Net/protocol after delete_namespaces" >&2 + echo "$ns_props" >&2 + exit 1 +fi + +echo ">> rd-clone-retry-semantics OK (replay is idempotent across a source resize; delete_namespaces accepted)" diff --git a/tests/operator-harness/replay/rd-clone-retry-semantics.yaml b/tests/operator-harness/replay/rd-clone-retry-semantics.yaml new file mode 100644 index 00000000..689169ff --- /dev/null +++ b/tests/operator-harness/replay/rd-clone-retry-semantics.yaml @@ -0,0 +1,104 @@ +name: rd-clone-retry-semantics +description: | + The clone endpoint's retry contract, at operator-CLI level. + + linstor-csi re-issues CreateVolume whenever a response is lost or + external-provisioner restarts, so `rd clone ` run twice has + to answer the same way twice. A finished clone is a copy of a + point-in-time and owes the source nothing afterwards -- including + after the source grows. + + That last part is what this replay exists for. A staleness guard that + compared the internal snapshot against the LIVE source turned an + ordinary `vd set-size` into a permanent refusal of every later retry: + the snapshot is named deterministically and outlives the clone by + design (delta row 82), the restore marker survives, and the refusal's + own correction -- delete the snapshot -- cannot be followed, because + that snapshot is the origin the existing clone depends on. + + Steps: + 1. rd create + vd create + r create --auto-place=2 + 2. wait the source UpToDate + 3. clone; wait the target's replicas to converge + 4. clone AGAIN with the same argv -- the replay, exit 0 + 5. vd set-size on the source, then clone a third time -- exit 0 + delete_namespaces is deliberately absent here: the runner executes + `linstor` argv, and the clone verb has no flag for it (1.31.0 offers + --external-name, --use-zfs-clone, --volume-passphrase, --layer-list and + --resource-group). It is a WIRE field, carried by golinstor's + GenericPropsModify, so the L6 cell drives it over raw REST instead. + + The byte-level data-plane assert lives in the sibling cell + tests/e2e/cli-matrix/rd-clone-vd-data-plane.sh; this replay pins the + verb and convergence shape the runner can express. + + Teardown order matters: the clones depend on the internal source + snapshots `clone-` and `rd delete` on the source is blocked + while snapshots exist, so the clone RDs go first, then the snapshots, + then the source. + +prerequisites: + min_nodes: 2 + storage_pool: stand + +vars: + # Short fixed name: the clone suffixes and the internal snapshot names + # clone-{{rd}}-c1 must stay under LINSTOR's 48-char ceiling. + rd: replay-clretry + sp: stand + +steps: + - name: create-rd + cmd: ["resource-definition", "create", "{{rd}}"] + expect_exit: 0 + - name: create-vd + cmd: ["volume-definition", "create", "{{rd}}", "64M"] + expect_exit: 0 + - name: auto-place-2 + # all_uptodate rather than replica_count, and no `resource list` step + # after it: the awaits poll the API directly, while a CLI list issued in + # the first seconds after a replica is stamped crashes the official + # python client outright (layer_object reports type DRBD before the drbd + # payload is filled, and the client indexes it unguarded). That window is + # not this workflow's subject and is not specific to cloning -- a plain + # `resource create` shows it too -- so the workflow waits on state + # instead of racing a renderer. + cmd: ["resource", "create", "--auto-place", "2", "--storage-pool={{sp}}", "{{rd}}"] + expect_exit: 0 + await: + kind: all_uptodate + rd: "{{rd}}" + timeout_s: 240 + - name: clone + cmd: ["resource-definition", "clone", "{{rd}}", "{{rd}}-c1"] + expect_exit: 0 + await: + kind: all_uptodate + rd: "{{rd}}-c1" + timeout_s: 240 + - name: replay-the-finished-clone + # The retry CSI issues when a response is lost. Same argv, same answer. + # The runner's resend tolerance is off: the refusal of a target that is + # not this clone reads "already exists" too, and would pass the step. + cmd: ["resource-definition", "clone", "{{rd}}", "{{rd}}-c1"] + expect_exit: 0 + tolerate_resend_409: false + - name: grow-the-source + cmd: ["volume-definition", "set-size", "{{rd}}", "0", "128M"] + expect_exit: 0 + - name: replay-after-the-source-grew + # The regression this replay is written for: a finished clone is a + # copy of a point-in-time, so a later resize of the source cannot + # turn its retry into a refusal. + cmd: ["resource-definition", "clone", "{{rd}}", "{{rd}}-c1"] + expect_exit: 0 + tolerate_resend_409: false + +teardown: + - cmd: ["resource-definition", "delete", "{{rd}}-c1"] + - cmd: ["snapshot", "delete", "{{rd}}", "clone-{{rd}}-c1"] + - cmd: ["resource-definition", "delete", "{{rd}}"] + +invariants: + # Prefix-match on vars.rd also covers the "-c1" leftover. + - no_orphans From 13c3274aaf2c1397eb0e1c9eebef547b892a34c7 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 7 Oct 2026 10:01:19 +0200 Subject: [PATCH 4/4] feat(clone): reap the internal snapshot a clone took on its source A clone of a source with volumes takes `clone-` on the source and restores from it, and nothing ever deleted that snapshot. Deleting the clone left it behind, and the source could then never be deleted again through the API, which refuses a definition that has snapshots. Every CSI clone goes through this path. `rd d` of a clone now reaps that snapshot, on both the REST and the CLI door. Ownership is a prop the clone path stamps on the snapshot, never its name, so an operator's snapshot called `clone-` is never touched. The snapshot is visible in `s l`, so another definition can have been restored from it, and that definition keeps reading it: new replicas are pinned to its nodes and restored from it by name. A dependent appears in two steps, the snapshot read and the definition create, so no list on the reap's side alone can see one between them. The reap marks the snapshot, then lists dependents past the cache, and only then deletes; a restore creates its definition, then reads the snapshot back past the cache and withdraws on a live mark, on a snapshot whose delete is in flight, or on one that is gone. Whichever of the list and the create comes second sees the other. A snapshot the reap keeps has the mark taken off again. The reap runs detached from its caller on a bounded budget, since the delete has already been reported, and a mark left by a process that died expires. A delete that was issued is read off the snapshot itself: the store now surfaces DELETE on a snapshot a finalizer still holds, the way it does for resources, and the snapshot list no longer stamps SUCCESSFUL beside it. A later `rd d` of the source names what is still there: a snapshot safe to delete, one kept because a named definition restores from it, and one that looks like a clone's but was never stamped. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- docs/cli-parity-known-deltas.md | 4 +- internal/cli/definition.go | 17 +- internal/cli/rd_delete_clone_snapshot_test.go | 112 +++++ internal/cli/snapshot.go | 28 +- .../cli/snapshot_restore_withdraw_test.go | 81 ++++ internal/cli/write.go | 26 ++ pkg/api/v1/snapshot.go | 3 + pkg/rest/cache_invalidation_bug_124_test.go | 4 + pkg/rest/rd_clone.go | 16 + pkg/rest/rd_delete_clone_snapshot_test.go | 315 ++++++++++++++ pkg/rest/resource_definitions.go | 86 +++- pkg/rest/snapshot_restore.go | 45 ++ pkg/rest/snapshot_restore_withdraw_test.go | 196 +++++++++ pkg/rest/snapshots.go | 8 +- pkg/rest/snapshots_test.go | 40 ++ pkg/store/clone_snapshot.go | 405 ++++++++++++++++++ pkg/store/clone_snapshot_mark_test.go | 212 +++++++++ pkg/store/clone_snapshot_test.go | 48 +++ pkg/store/clone_snapshot_uncached_test.go | 200 +++++++++ pkg/store/inmemory_resource_definition.go | 5 + pkg/store/k8s/deleting_flag_internal_test.go | 28 ++ pkg/store/k8s/rd_list_uncached_test.go | 68 +++ pkg/store/k8s/resource_definitions.go | 38 +- pkg/store/k8s/snapshots.go | 8 + pkg/store/local_props.go | 18 +- pkg/store/store.go | 7 + 26 files changed, 1974 insertions(+), 44 deletions(-) create mode 100644 internal/cli/rd_delete_clone_snapshot_test.go create mode 100644 internal/cli/snapshot_restore_withdraw_test.go create mode 100644 pkg/rest/rd_delete_clone_snapshot_test.go create mode 100644 pkg/rest/snapshot_restore_withdraw_test.go create mode 100644 pkg/store/clone_snapshot.go create mode 100644 pkg/store/clone_snapshot_mark_test.go create mode 100644 pkg/store/clone_snapshot_test.go create mode 100644 pkg/store/clone_snapshot_uncached_test.go create mode 100644 pkg/store/k8s/rd_list_uncached_test.go diff --git a/docs/cli-parity-known-deltas.md b/docs/cli-parity-known-deltas.md index c6b55014..84fd37c2 100644 --- a/docs/cli-parity-known-deltas.md +++ b/docs/cli-parity-known-deltas.md @@ -50,12 +50,12 @@ Row IDs match the command-catalogue indexes used by `cli-parity-refresh.sh` (see | 80 | `r td --migrate-from ` (disklessOnRemaining on the vacated source) | BEHAVIOR | permanent | Upstream-issue U435 (P2, "No diskless replica recreated on source after `--migrate-from` despite disklessOnRemaining"). RG `disklessOnRemaining` IS implemented in blockstor — the placer's `placeDisklessOnRemaining` (`pkg/placer/placer.go`) fans a DISKLESS witness onto every healthy node not already hosting a replica, on the AUTOPLACE path (pinned by `pkg/placer/placer_test.go::TestPlaceDisklessOnRemaining`). The divergence is scoped strictly to the `--migrate-from` path: the `ResourceMigrationReconciler` (`internal/controller/resource_migration_controller.go`) prunes the source's DISKFUL Resource CRD once the destination is UpToDate but does NOT itself re-stamp a `disklessOnRemaining` witness on the vacated source — it relies on the auto-tiebreaker machinery instead. On a cluster where the post-migration replica set has even parity (e.g. the canonical 2-diskful result), `ensureTiebreaker` immediately RE-OCCUPIES the vacated source node with a DISKLESS `TIE_BREAKER` Resource — the same node-presence outcome `disklessOnRemaining` intends (every node retains DRBD-network presence / a quorum voter; see row 78). The witness's FLAG differs (`TIE_BREAKER` vs a plain `disklessOnRemaining` DISKLESS stub) but the operator-visible effect — the source node is left with a diskless replica, not "nothing" (the U435 symptom) — holds. Where parity is odd (no auto-TB needed), the vacated node is genuinely left bare, matching upstream's pre-fix behaviour for that case. CSI/Cozystack never sets `disklessOnRemaining` on its RGs (it relies on auto-TB for quorum), so the migrate-path gap is operator-invisible for the target workload. The auto-TB re-occupation of the migrate source is pinned by `tests/operator-harness/replay/toggle-disk-migrate-from.yaml` (`assert-src-removed` awaits `replica_diskless` on the source node) + `tests/e2e/cli-matrix/toggle-disk-migrate-from.sh`. No fix required; documented so a future `disklessOnRemaining + --migrate-from` report maps here. | | 81 | `rd l` / `rd s` (Layers column / `layer_data` on unplaced RDs) | WIRE_SHAPE | permanent | Bug 349: the python CLI's `rd l` Layers column `,`-joins `layer_data[].type`, but blockstor's k8s store persists only `Spec.LayerStack` — so the read path re-synthesises a flat `layer_data: [{"type":"DRBD"},{"type":"STORAGE"}]` on EVERY RD GET / list response (`stampRDLayerDataFromStack`, pkg/rest/resource_definitions.go), falling back to the upstream default stack `DRBD,STORAGE` when the stored stack is empty, so the column never renders blank. Upstream LINSTOR 1.33.2 emits `layer_data` on RD responses only once DRBD layer data actually exists (resources placed → port/secret allocated), and then with a volatile `data` payload (`peer_slots`, `port`, `secret`, …) blockstor never synthesises; a bare `rd c` and an RD with only a volume-definition both omit the field entirely (verified live against the dev-stand oracle 2026-06-12). Operator-visible effect: BS renders the Layers column for unplaced RDs where upstream renders an empty cell — BLOCKSTOR_SUPERSET; once resources exist both sides list the same `type` sequence (the CLI renders only the types, never upstream's `data` payload). linstor-csi / piraeus-operator do not gate on RD `layer_data`. The L3 oracle-replay harness tolerates exactly this field on RD-shaped objects (scoped per-field drop in tests/contract/normalize.go keyed on `resource_group_name`; resources' `layer_object` / volumes' `layer_data_list` still diff). Pinned by `tests/contract/normalize_test.go::TestNormalizeRDLayerDataDropped` (L1) + the 4 RD traces in `tests/contract/testdata/oracle/` (L3). | -| 82 | `rd clone` data plane (`use_zfs_clone` vs `zfs send\|recv`) | BEHAVIOR | permanent | Bug-020. Upstream LINSTOR clones a VD-bearing RD by internal snapshot + either `zfs clone` (when the request carries `use_zfs_clone=true`, golinstor v0.58+/linstor-csi) or `zfs send \| zfs recv` (default, fully independent copy). blockstor's clone routes through the snapshot-restore machinery: internal snapshot `clone-` + `BlockstorRestoreFromSnapshot` marker, whose ZFS provider materialises the target with `zfs clone`. Bug 038: clone/restore replicas are stamped on the snapshot-holding nodes in the SOURCE replica's storage pool (upstream restore semantics, oracle-verified), and the placer refuses to land later replicas of a restore-marked RD on a different backend — the snapshot stream formats are not interchangeable (FILE_THIN stream into `zfs recv` loops on bad magic). Upstream `rd clone` likewise operates on the source's own pools (it refuses unsupported source pools: "Clone source contains unsupported storage pools"). Consequences accepted: (a) `use_zfs_clone=true` — the linstor-csi case — gets exactly the requested semantics; (b) `use_zfs_clone=false`/absent ALSO lands on the snapshot-clone path instead of an independent full copy, so same-node clone targets stay dependent on the origin snapshot (the snapshot is visible in `linstor s l` and must outlive the clone); (c) sources on non-snapshot-capable (thick) pools refuse the clone with an actionable envelope where upstream would full-copy. Pinned by `pkg/rest/clone_use_zfs_clone_bug020_test.go` + `pkg/rest/clone_placement_bug038_test.go` + `pkg/placer/restore_source_constraints_bug038_test.go` (L1) + `tests/integration` Group J `CSICreateVolumeFromClone` (Tier 2). | +| 82 | `rd clone` data plane (`use_zfs_clone` vs `zfs send\|recv`) | BEHAVIOR | permanent | Bug-020. Upstream LINSTOR clones a VD-bearing RD by internal snapshot + either `zfs clone` (when the request carries `use_zfs_clone=true`, golinstor v0.58+/linstor-csi) or `zfs send \| zfs recv` (default, fully independent copy). blockstor's clone routes through the snapshot-restore machinery: internal snapshot `clone-` + `BlockstorRestoreFromSnapshot` marker, whose ZFS provider materialises the target with `zfs clone`. Bug 038: clone/restore replicas are stamped on the snapshot-holding nodes in the SOURCE replica's storage pool (upstream restore semantics, oracle-verified), and the placer refuses to land later replicas of a restore-marked RD on a different backend — the snapshot stream formats are not interchangeable (FILE_THIN stream into `zfs recv` loops on bad magic). Upstream `rd clone` likewise operates on the source's own pools (it refuses unsupported source pools: "Clone source contains unsupported storage pools"). Consequences accepted: (a) `use_zfs_clone=true` — the linstor-csi case — gets exactly the requested semantics; (b) `use_zfs_clone=false`/absent ALSO lands on the snapshot-clone path instead of an independent full copy, so same-node clone targets stay dependent on the origin snapshot (the snapshot is visible in `linstor s l` and must outlive the clone; `rd d ` reaps it from the source through both the REST and the CLI door, since the dependency ends with the target, and until it did the source could not be deleted at all, because `rd d` refuses a definition that has snapshots. Ownership is the `Blockstor/CloneSnapshotOf` prop the clone path stamps on the snapshot, never its name, so an operator's snapshot that happens to be called `clone-` is never reaped; a snapshot another definition was restored from is kept, asked of the API server rather than a cache, after the snapshot is marked as being reaped, and a restore reads its snapshot back once its definition exists and withdraws on that mark or on a snapshot that is gone; the reap outlives its caller, and a mark it leaves anyway expires after five minutes; and one taken by a version before the prop existed is left for the operator. A later `rd d` of the source names each kind: one safe to delete, one kept for a named restore, one that looks like a clone's but was never stamped); (c) sources on non-snapshot-capable (thick) pools refuse the clone with an actionable envelope where upstream would full-copy. Pinned by `pkg/rest/clone_use_zfs_clone_bug020_test.go` + `pkg/rest/clone_placement_bug038_test.go` + `pkg/placer/restore_source_constraints_bug038_test.go` (L1) + `tests/integration` Group J `CSICreateVolumeFromClone` (Tier 2). | | 83 | `rg spawn-resources` on an over-committed RG (place_count > available nodes) | BEHAVIOR | permanent | Corner D1. Upstream LINSTOR fails `spawn-resources` SHORT when the RG's place_count exceeds the placeable node count: it returns `FAIL_NOT_ENOUGH_NODES` (ret_code 996, "Not enough available nodes") and places NOTHING. blockstor instead takes a DEFERRED, best-effort autoplace path: it spawns the RD + VDs, places as many diskful replicas as the topology allows (e.g. 3 of 7 on a 3-node cluster), and surfaces the shortfall as an INFO in the SUCCESS envelope (`resource definition spawned, autoplace deferred: : not enough candidate storage pools: placed N of M`, exit 0). The `RGRebalanceReconciler` (`internal/controller/rg_rebalance_controller.go`) then tops the replica count back up additively once more nodes appear. The over-commit `rg create` itself is ACCEPTED by both controllers (parity — never an early create-time error). Rationale: the deferred path lets the CSI external-provisioner retry loop converge on a created RD instead of hard-failing on a cluster that is one node away from satisfying the request, and is strictly additive (scale-down stays an explicit `r d`). Switching spawn back to the upstream 996 short-fail would be a deliberate API change, not a silent regression. Pinned by `pkg/rest/spawn_test.go::TestSpawnImpossiblePlacementReturnsActionableError` + `TestSpawnPartialFlagAllowsShortPlacement` (L1) and `tests/e2e/cli-matrix/rg-c-overcommit-spawn-defers.sh` (L6). | | 84 | `r c ` (tiebreaker→diskful promotion runs full SyncTarget, not skip-sync) | BEHAVIOR | permanent | Promoting a tiebreaker (diskless witness) to diskful runs a FULL SyncTarget on the promoted node instead of the upstream skip-sync. DELIBERATE: skip-sync would require BS to pre-stamp a matching day0 GI and let the fresh replica win the auto-primary election — but a diskful peer already holds data (`anyDiskfulPeerHasData == true`, `pkg/dispatcher/dispatcher.go`), so force-priming the fresh replica mints an UNRELATED Current UUID; the data-bearing peer then declines the handshake (`uuid_compare()=unrelated-data`, "Unrelated data, aborting!") and the pair wedges in mutual StandAlone that never auto-recovers (the respawn-StandAlone P0). BS therefore gates the auto-primary seed on `!anyDiskfulPeerHasData(peers)` and `!rdInitialized(rd)`: with a data-bearing peer present the promoted replica comes up Inconsistent and SyncTargets the real bytes off the peer. Data converges correctly; the only cost is a full resync of the volume — the safe trade vs. a StandAlone wedge. Upstream's skip-sync-on-promotion is not reproducible without reintroducing the wedge. Pinned by `tests/e2e/cli-matrix/r-c-over-tiebreaker-skip-sync.sh` (L6, asserts SyncTarget→UpToDate convergence + Bug 348 SyncSource shape) + the dispatcher auto-primary gate tests (`pkg/dispatcher/dispatcher_test.go`, respawn-StandAlone wedge regression). | | 85 | `r l` State column — bare `SyncSource`/`SyncTarget` literal (no `(NN%)` suffix) when OutOfSyncKib<=0 | WIRE_SHAPE | permanent | A replica actively resyncing renders `SyncTarget(NN%)` / `SyncSource(NN%)` WHILE there is data to copy. When the per-volume `OutOfSyncKib` is <= 0 (or the VD size is unknown), BS DELIBERATELY renders the BARE literal `SyncSource` / `SyncTarget` with NO `(NN%)` suffix: `withSyncPercent` (pkg/rest/resources.go, called from `annotateSyncProgress`) short-circuits on `outOfSyncKib<=0` rather than emit a misleading `(0%)`/`(100%)`. The drbd replication-state literal can be observed for a brief window before/after the peer-device OutOfSyncKib counter carries a non-zero value, so a bare Sync* token is a real and intended shape. The REGRESSION guarded against is the opposite — a TERMINAL `UpToDate` must never carry a `(NN%)` suffix (Bug A). So the contract is: a Sync* token may render bare OR with `(NN%)`, but a terminal disk_state never carries a percent. Pinned by `tests/e2e/cli-matrix/r-l-conns-shapes.sh` sub-test D (L6) + `annotateSyncProgress`/`withSyncPercent` unit coverage (pkg/rest, Bug 348 + Bug A/B). | | 86 | `rd clone --external-name` / `--volume-passphrase` / a `--layer-list` that changes the source's layer set | MISSING_FEATURE | permanent | `external_name` gives the clone an identity of its own upstream and `volume_passphrases` carries the LUKS keys for its volumes; blockstor names a clone by `name` and takes its keys from the cluster passphrase, so honouring neither is possible and both are refused with an explicit 501 rather than accepted and dropped. Every refusal on this endpoint, the 501 included, goes out in the shape the caller decodes: the CloneStarted object for python-linstor, which names itself `PythonLinstor/…` and reads the clone answer as that object whatever the status, and an ApiCallRc array for every other client, since golinstor decodes a non-2xx body as nothing else (linstor-csi's 404s reach it bare either way) — a clone that reports success under a different name, or with keys the caller does not hold, is the failure mode `src_snap_name` is already refused for. `layer_list` IS honoured, with one refusal: on a source that has volumes the requested stack must be the source's, compared as a set (order is the stack's own and case folds). The clone data plane restores the source's bytes and brings the layer stack up over them, in that order, and every layer's bring-up writes to the device — LUKS formats a device carrying no header, DRBD stamps `meta-disk internal` metadata with `create-md --force` — so a layer added here lands across the bytes just restored and the clone reports COMPLETE over them, while a layer dropped leaves the target reading data the missing layer wrote. A source with no recorded stack means the upstream default `[DRBD, STORAGE]`, which is what linstor-csi sends, so the CSI clone path is unaffected. Upstream clones the layer list with the volume, re-encrypting LUKS under the cluster key. A volume-less source has no bytes to lose and keeps taking any stack. A clone resumed over a leftover also keeps that leftover's shape: a retry naming a different `resource_group` or stack is refused rather than answered 201 with the request's shape dropped, and a leftover internal snapshot that has fallen behind the source (a volume added, or resized) is refused rather than cloned from. Pinned by `pkg/rest/rd_clone_golinstor_shape_test.go` + `pkg/rest/rd_clone_client_envelope_test.go` + `pkg/rest/rd_clone_layer_stack_test.go` + `pkg/rest/rd_clone_resume_test.go` + `pkg/rest/rd_clone_leftover_shape_test.go` + `pkg/rest/rd_clone_server_owned_props_test.go` + `pkg/rest/rd_modify_server_owned_props_test.go` + `internal/cli/snapshot_restore_prepared_test.go` (L1) and `tests/e2e/cli-matrix/rd-clone-retry-semantics.sh` (L6, stand-only: `make e2e-list` covers `tests/e2e/*.sh` and no CI lane runs the cli-matrix cells) + `tests/operator-harness/replay/rd-clone-retry-semantics.yaml` (L7). `rd modify` and `rd set-property` / `delete-property` refuse the same server-owned props, with 400 and an error respectively, since a `delete_namespaces` of `Blockstor` mid-restore took the adoption mark with it, and `rd create` refuses a definition carrying one with 400. Prop edits on a clone (`override_props`, `delete_props`, `delete_namespaces`) that would rewrite or remove a prop blockstor sets on the clone itself, the restore marker, the record of the snapshot's volumes, the rollback mark or the adoption mark, are refused with 400 before anything is written, rather than applied after the marker and leaving the satellite to bring the volumes up blank; linstor-csi's own `delete_props` edit touches none of them. | -| 87 | `rd clone` / `s resource restore` retry over an unfinished target | BEHAVIOR | permanent | This is the data path: a clone of a source with volumes, and every restore. A clone of a volume-less source is a plain copy with no marker, so a repeat of it stays the already-exists refusal upstream gives. Both endpoints stamp `BlockstorRestoreFromSnapshot` with the target definition and hydrate its volumes and place its replicas afterwards, so a target carrying the marker says the operation STARTED, not that it finished. A repeat under the same name RESUMES it — every step tolerates an object a previous attempt already created — and says so (`… completed on retry`); upstream has no equivalent notion, a repeat there is a plain already-exists failure. This is what makes CSI's CreateVolume idempotent: external-provisioner has no other way to make progress after a partial failure, and answering success on the marker alone would leave CSI seeing a ready volume nothing ever finishes. A definition under that name that carries no marker is taken when it is in the state LINSTOR's own sequence leaves (`rd c`, `s vd restore`, then `s rsc restore`): its volumes exactly the snapshot's and no live replica. linstor-csi restores every volume that way, creating the definition itself first, and the marker is stamped on it at that point. What stays a refusal, each before anything is written to the definition: any other definition under that name this operation did not create and that is not prepared, meaning its volumes are not the snapshot's or it holds a live replica (409); a prepared definition whose layer stack is not the snapshot source's, compared as a set, since the restore would put the source's bytes under layers it never had or without layers it had (409); a prepared definition, or a leftover still to be finished or one the request places replicas into, whose every replica is still being deleted, which placing over would race (409; a bare REST restore's replay over a finished leftover still answers 201 while its snapshot exists, since it places none); a prepared definition or a leftover parented to a resource group that no longer exists (409, or 500 asking for a retry when the group cannot be read); this operation's own leftover carrying the DELETE flag, since finishing it races the tear-down reaping what it writes (409); a leftover whose earlier attempt's rollback gave up, which may hold less than the operation intended, with the way out named (409); a leftover holding a volume the snapshot never recorded beside one it did (409); and a leftover whose creating attempt has started rolling it back (409). Placing a replica, after the definition and its volumes are written, refuses an operator's diskless replica already on a node the request places onto, which holds no copy of the data and is not counted as placed: a 409 on a retry over this operation's own leftover, and on a fresh restore a failed placement, which rolls the new definition back and answers 500, neither with a FAIL_EXISTS band. The controller's tie-breaker witness there is promoted to the replica the request asked for instead, as autoplace promotes one, and its volumes are restored from the snapshot like any other replica of the target; a witness no storage pool can be found for is left as it is and refused the same way, naming `linstor resource create --storage-pool ` to give it one. Once the marker is on a prepared definition it stays: a concurrent retry that finds it resumes and places the replica, and the request that wrote it meets that replica as one already placed, never as a reason to take the marker back off. The window this leaves is a replica placed by hand between the judgement and the marker, which is then judged as part of the restore; on a target prepared for exactly this restore nothing else places one in that moment. A replica the operation would place that is still being deleted on its node fails the request rather than counting as placed (a 409 over a leftover this request adopted, and a 500 with the rollback's answer over one it created), with no FAIL_EXISTS_RSC band on either. Two restores of different snapshots into one prepared definition do not both take it: the second finds the first's marker and is refused. The CLI door settles the parent group, and the nodes and pools the replicas go to, before it writes the definition or the marker, so a request refused for its own shape, or over a prepared definition whose group is gone, leaves that definition unmarked; and a CLI placement that fails after the marker is finished by running the same command again, unless a replica already landed: the re-run then judges it finished, so the failure names the replicas that did not land and the `blockstor resource create` that places them. The CLI judges its own leftover with the same assessment and gate order the REST door uses (a tear-down, then a volume the restore would not have written, then a parent group that is gone, then a rollback that gave up): a leftover with the snapshot's volumes and one replica with a disk is finished and left alone rather than re-placed on a node the operator emptied, and only one with no such replica gets its placement finished, under the same adoption mark the REST door takes, so a rollback by the attempt that created it yields. A definition already under the name is judged before anything about the source is planned, so a finished restore is left alone even once the source can no longer be planned from. Replicas are counted past the cache on both doors, and a rollback still in progress is refused as one still running rather than as one that gave up. A clone resume whose internal snapshot is gone is refused once the leftover holds a volume with a replica that has a disk, since finishing it from a new snapshot would join two moments of the source; a leftover with no volume or no replica is finished from the new snapshot like a fresh clone, unless its volumes do not fit the source as it is now (one smaller than the source's, or one the source no longer has), which is refused with `delete and clone again` before any snapshot is taken. The volumes and replicas of a prepared definition, and a colliding replica, are read from the API server, not a cache, and so is the definition the request judges right after stamping its marker: linstor-csi's volume restore and the resource restore after it can be served by different replicas of the server. A finished clone or restore is judged by its volumes and, where the operation placed replicas, by holding one with a disk, never by where they are now: a replay after a node was evacuated or the target scaled down answers success and leaves placement alone. A finished clone is a copy of a point-in-time and its replay answers the same way whatever happens to the source afterwards — including a `vd set-size`, which a guard comparing the internal snapshot to the live source turned into a permanent refusal of every retry. Pinned by `pkg/rest/rd_clone_idempotency_test.go` + `pkg/rest/rd_clone_replay_test.go` + `pkg/rest/rd_clone_leftover_shape_test.go` + `pkg/rest/snapshot_restore_idempotency_test.go` + `pkg/rest/snapshot_restore_replay_test.go` + `pkg/rest/snapshot_restore_csi_flow_test.go` + `pkg/rest/snapshot_restore_prepared_target_test.go` + `pkg/rest/snapshot_restore_prepared_cache_test.go` + `pkg/rest/snapshot_restore_resume_gates_test.go` + `pkg/rest/leftover_gates_test.go` + `pkg/rest/rollback_handshake_failclosed_test.go` + `pkg/store/restore_target_test.go` + `internal/cli/snapshot_restore_prepared_test.go` (L1) and `tests/e2e/cli-matrix/rd-clone-retry-semantics.sh` (L6, stand-only: `make e2e-list` covers `tests/e2e/*.sh` and no CI lane runs the cli-matrix cells) + `tests/operator-harness/replay/rd-clone-retry-semantics.yaml` (L7). Every marker is written with `Blockstor/RestoreVolumes`, the volumes the snapshot held, so a leftover whose snapshot is gone (an operator deleted it) is still judged against them: a clone that restored only some of its volumes is unfinished, not complete. A leftover from before the record existed carries none and is judged on its own volumes, as before. A CLI rollback that could not delete takes its in-progress mark back off so the next run finishes the definition, and a CLI re-run over a finished restore says so on stdout. The CLI's rollback of a definition it created holds the same handshake as REST before deleting: a definition another request adopted is left to it, and one whose mark could not be written or whose adoption could not be read is not deleted. A rollback after a failed clone or restore also leaves a definition that already reads as finished, holding every volume and, where the operation places replicas, one replica with a disk: the clone status poll answers such a definition complete without writing anything, which can happen while the attempt that created it is still stamping its last replica, so deleting it afterwards took a volume the driver had been told exists. A replay takes the adoption mark before its 201, so the rollback yields to it like to any adopter, whatever the request placed. The rollback over a group deleted under the operation is the exception: it removes a finished definition too, since one parented to a group that is gone is what it exists to remove, so a status poll that answered complete a moment before it starts is followed by the definition going away; a POST replay's mark makes it yield. That attempt fails, and its retry answers for what it left; a restore that asked for replicas on named nodes names the ones it never placed, with the `linstor resource create` that places them, since the retry judges the definition finished and places nothing. A clone's prop edits (`override_props`, `delete_props`, `delete_namespaces`) land with the definition when it is created, so no leftover carries a prop the caller asked to remove: linstor-csi removes `Aux/csi-provisioning-completed-by` on every clone and takes a definition carrying it, with volumes of the right size, as one already provisioned. The clone status poll asks a finished target what the POST's replay asks before its 201: one carrying the DELETE flag, parented to a group that is gone, or carrying a rollback in progress or one that gave up is answered 404, which sends the driver back to the POST that refuses it, and a group that cannot be read is a 500. A replay of a finished clone is answered before the source is read, so it survives the source being deleted or losing its volumes. A retry after the snapshot was deleted, the cleanup that follows a finished restore, is judged against the volumes the definition recorded when it was restored: a finished one answers 201 (the CLI exits 0) without reading the snapshot or the source, an unfinished one is refused with `delete and restore it from another snapshot`, and a target restored before the record existed keeps the 404 it got. A restored or cloned definition is stamped with the source's data-plane layer stack (its own stored stack, or DRBD over STORAGE when it has none): the dispatcher reads only the definition's stack, so a target group with another stack never changed how the bytes are brought up, but the control plane reads the group's stack for a definition stored without one, and the stamp keeps the two in step. A REST replay of a finished restore checks its node list only while the snapshot exists; once it is gone the replay is judged by the target alone. A bare restore counts as finished without a replica only while its snapshot exists: once the snapshot is gone, a target with no replica holds its data nowhere and is refused with `delete and restore it from another snapshot`, as the CLI refuses it. A clone that names no layer stack takes the source's and needs the cluster passphrase when that stack carries LUKS, as one that names it does; the check runs after the replay of a finished clone, which writes no new LUKS volume. A target already under the name is judged, hydrated and placed under the name it is stored with, whatever case the request spells it in: its replicas are selected by that name exactly, and judged under another spelling a finished clone or restore read as having none; a read of that name that fails is the 500 below. A target that cannot be read is answered 500 asking for a retry, not taken as absent: the create that followed collided and was adopted past every judgement of a leftover. The CLI refuses a snapshot that recorded no volumes before it writes or resumes anything, as the REST door does. A rollback that leaves a definition whole and cannot take its in-progress mark back off says so and names `linstor resource-definition set-property BlockstorRollbackAbandoned` with no value (python-linstor's delete; the blockstor CLI also has `delete-property`), which `rd modify` (`delete_props`) and the CLI accept as the one edit of a server-owned prop an operator may make, and only while the mark sits on a definition the rollback left whole (`in-progress`, or `snapshots` / `read-snapshots`, where it stopped before touching anything): one naming a later step sits on a definition the rollback took partly apart, and stays; setting the mark stays refused, and an empty `override_props` value of it is read as the delete. The refusal over an in-progress mark names the clear only once the rollback has had its time; a clear while it still runs stops that rollback before it deletes anything, since the rollback checks its own mark is still there before its first delete. A CLI rollback that leaves the target in place names the `blockstor snapshot resource restore` that finishes it, including for `rd clone`, whose own re-run is refused over a target that exists. | +| 87 | `rd clone` / `s resource restore` retry over an unfinished target | BEHAVIOR | permanent | This is the data path: a clone of a source with volumes, and every restore. A clone of a volume-less source is a plain copy with no marker, so a repeat of it stays the already-exists refusal upstream gives. Both endpoints stamp `BlockstorRestoreFromSnapshot` with the target definition and hydrate its volumes and place its replicas afterwards, so a target carrying the marker says the operation STARTED, not that it finished. A repeat under the same name RESUMES it — every step tolerates an object a previous attempt already created — and says so (`… completed on retry`); upstream has no equivalent notion, a repeat there is a plain already-exists failure. This is what makes CSI's CreateVolume idempotent: external-provisioner has no other way to make progress after a partial failure, and answering success on the marker alone would leave CSI seeing a ready volume nothing ever finishes. A definition under that name that carries no marker is taken when it is in the state LINSTOR's own sequence leaves (`rd c`, `s vd restore`, then `s rsc restore`): its volumes exactly the snapshot's and no live replica. linstor-csi restores every volume that way, creating the definition itself first, and the marker is stamped on it at that point. What stays a refusal, each before anything is written to the definition: any other definition under that name this operation did not create and that is not prepared, meaning its volumes are not the snapshot's or it holds a live replica (409); a prepared definition whose layer stack is not the snapshot source's, compared as a set, since the restore would put the source's bytes under layers it never had or without layers it had (409); a prepared definition, or a leftover still to be finished or one the request places replicas into, whose every replica is still being deleted, which placing over would race (409; a bare REST restore's replay over a finished leftover still answers 201 while its snapshot exists, since it places none); a prepared definition or a leftover parented to a resource group that no longer exists (409, or 500 asking for a retry when the group cannot be read); this operation's own leftover carrying the DELETE flag, since finishing it races the tear-down reaping what it writes (409); a leftover whose earlier attempt's rollback gave up, which may hold less than the operation intended, with the way out named (409); a leftover holding a volume the snapshot never recorded beside one it did (409); and a leftover whose creating attempt has started rolling it back (409). Placing a replica, after the definition and its volumes are written, refuses an operator's diskless replica already on a node the request places onto, which holds no copy of the data and is not counted as placed: a 409 on a retry over this operation's own leftover, and on a fresh restore a failed placement, which rolls the new definition back and answers 500, neither with a FAIL_EXISTS band. The controller's tie-breaker witness there is promoted to the replica the request asked for instead, as autoplace promotes one, and its volumes are restored from the snapshot like any other replica of the target; a witness no storage pool can be found for is left as it is and refused the same way, naming `linstor resource create --storage-pool ` to give it one. Once the marker is on a prepared definition it stays: a concurrent retry that finds it resumes and places the replica, and the request that wrote it meets that replica as one already placed, never as a reason to take the marker back off. The window this leaves is a replica placed by hand between the judgement and the marker, which is then judged as part of the restore; on a target prepared for exactly this restore nothing else places one in that moment. A replica the operation would place that is still being deleted on its node fails the request rather than counting as placed (a 409 over a leftover this request adopted, and a 500 with the rollback's answer over one it created), with no FAIL_EXISTS_RSC band on either. Two restores of different snapshots into one prepared definition do not both take it: the second finds the first's marker and is refused. The CLI door settles the parent group, and the nodes and pools the replicas go to, before it writes the definition or the marker, so a request refused for its own shape, or over a prepared definition whose group is gone, leaves that definition unmarked; and a CLI placement that fails after the marker is finished by running the same command again, unless a replica already landed: the re-run then judges it finished, so the failure names the replicas that did not land and the `blockstor resource create` that places them. The CLI judges its own leftover with the same assessment and gate order the REST door uses (a tear-down, then a volume the restore would not have written, then a parent group that is gone, then a rollback that gave up): a leftover with the snapshot's volumes and one replica with a disk is finished and left alone rather than re-placed on a node the operator emptied, and only one with no such replica gets its placement finished, under the same adoption mark the REST door takes, so a rollback by the attempt that created it yields. A definition already under the name is judged before anything about the source is planned, so a finished restore is left alone even once the source can no longer be planned from. Replicas are counted past the cache on both doors, and a rollback still in progress is refused as one still running rather than as one that gave up. A clone resume whose internal snapshot is gone is refused once the leftover holds a volume with a replica that has a disk, since finishing it from a new snapshot would join two moments of the source; a leftover with no volume or no replica is finished from the new snapshot like a fresh clone, unless its volumes do not fit the source as it is now (one smaller than the source's, or one the source no longer has), which is refused with `delete and clone again` before any snapshot is taken. The volumes and replicas of a prepared definition, and a colliding replica, are read from the API server, not a cache, and so is the definition the request judges right after stamping its marker: linstor-csi's volume restore and the resource restore after it can be served by different replicas of the server. A finished clone or restore is judged by its volumes and, where the operation placed replicas, by holding one with a disk, never by where they are now: a replay after a node was evacuated or the target scaled down answers success and leaves placement alone. A finished clone is a copy of a point-in-time and its replay answers the same way whatever happens to the source afterwards — including a `vd set-size`, which a guard comparing the internal snapshot to the live source turned into a permanent refusal of every retry. Pinned by `pkg/rest/rd_clone_idempotency_test.go` + `pkg/rest/rd_clone_replay_test.go` + `pkg/rest/rd_clone_leftover_shape_test.go` + `pkg/rest/snapshot_restore_idempotency_test.go` + `pkg/rest/snapshot_restore_replay_test.go` + `pkg/rest/snapshot_restore_csi_flow_test.go` + `pkg/rest/snapshot_restore_prepared_target_test.go` + `pkg/rest/snapshot_restore_prepared_cache_test.go` + `pkg/rest/snapshot_restore_resume_gates_test.go` + `pkg/rest/leftover_gates_test.go` + `pkg/rest/rollback_handshake_failclosed_test.go` + `pkg/store/restore_target_test.go` + `internal/cli/snapshot_restore_prepared_test.go` (L1) and `tests/e2e/cli-matrix/rd-clone-retry-semantics.sh` (L6, stand-only: `make e2e-list` covers `tests/e2e/*.sh` and no CI lane runs the cli-matrix cells) + `tests/operator-harness/replay/rd-clone-retry-semantics.yaml` (L7). Every marker is written with `Blockstor/RestoreVolumes`, the volumes the snapshot held, so a leftover whose snapshot is gone (an operator deleted it, or it was a clone's internal one and was reaped) is still judged against them: a clone that restored only some of its volumes is unfinished, not complete. A leftover from before the record existed carries none and is judged on its own volumes, as before. A CLI rollback that could not delete takes its in-progress mark back off so the next run finishes the definition, and a CLI re-run over a finished restore says so on stdout. The CLI's rollback of a definition it created holds the same handshake as REST before deleting: a definition another request adopted is left to it, and one whose mark could not be written or whose adoption could not be read is not deleted. A rollback after a failed clone or restore also leaves a definition that already reads as finished, holding every volume and, where the operation places replicas, one replica with a disk: the clone status poll answers such a definition complete without writing anything, which can happen while the attempt that created it is still stamping its last replica, so deleting it afterwards took a volume the driver had been told exists. A replay takes the adoption mark before its 201, so the rollback yields to it like to any adopter, whatever the request placed. The rollback over a group deleted under the operation is the exception: it removes a finished definition too, since one parented to a group that is gone is what it exists to remove, so a status poll that answered complete a moment before it starts is followed by the definition going away; a POST replay's mark makes it yield. That attempt fails, and its retry answers for what it left; a restore that asked for replicas on named nodes names the ones it never placed, with the `linstor resource create` that places them, since the retry judges the definition finished and places nothing. A clone's prop edits (`override_props`, `delete_props`, `delete_namespaces`) land with the definition when it is created, so no leftover carries a prop the caller asked to remove: linstor-csi removes `Aux/csi-provisioning-completed-by` on every clone and takes a definition carrying it, with volumes of the right size, as one already provisioned. The clone status poll asks a finished target what the POST's replay asks before its 201: one carrying the DELETE flag, parented to a group that is gone, or carrying a rollback in progress or one that gave up is answered 404, which sends the driver back to the POST that refuses it, and a group that cannot be read is a 500. A replay of a finished clone is answered before the source is read, so it survives the source being deleted or losing its volumes. A retry after the snapshot was deleted, the cleanup that follows a finished restore, is judged against the volumes the definition recorded when it was restored: a finished one answers 201 (the CLI exits 0) without reading the snapshot or the source, an unfinished one is refused with `delete and restore it from another snapshot`, and a target restored before the record existed keeps the 404 it got. A restored or cloned definition is stamped with the source's data-plane layer stack (its own stored stack, or DRBD over STORAGE when it has none): the dispatcher reads only the definition's stack, so a target group with another stack never changed how the bytes are brought up, but the control plane reads the group's stack for a definition stored without one, and the stamp keeps the two in step. A REST replay of a finished restore checks its node list only while the snapshot exists; once it is gone the replay is judged by the target alone. A bare restore counts as finished without a replica only while its snapshot exists: once the snapshot is gone, a target with no replica holds its data nowhere and is refused with `delete and restore it from another snapshot`, as the CLI refuses it. A clone that names no layer stack takes the source's and needs the cluster passphrase when that stack carries LUKS, as one that names it does; the check runs after the replay of a finished clone, which writes no new LUKS volume. A target already under the name is judged, hydrated and placed under the name it is stored with, whatever case the request spells it in: its replicas are selected by that name exactly, and judged under another spelling a finished clone or restore read as having none; a read of that name that fails is the 500 below. A target that cannot be read is answered 500 asking for a retry, not taken as absent: the create that followed collided and was adopted past every judgement of a leftover. The CLI refuses a snapshot that recorded no volumes before it writes or resumes anything, as the REST door does. A rollback that leaves a definition whole and cannot take its in-progress mark back off says so and names `linstor resource-definition set-property BlockstorRollbackAbandoned` with no value (python-linstor's delete; the blockstor CLI also has `delete-property`), which `rd modify` (`delete_props`) and the CLI accept as the one edit of a server-owned prop an operator may make, and only while the mark sits on a definition the rollback left whole (`in-progress`, or `snapshots` / `read-snapshots`, where it stopped before touching anything): one naming a later step sits on a definition the rollback took partly apart, and stays; setting the mark stays refused, and an empty `override_props` value of it is read as the delete. The refusal over an in-progress mark names the clear only once the rollback has had its time; a clear while it still runs stops that rollback before it deletes anything, since the rollback checks its own mark is still there before its first delete. A CLI rollback that leaves the target in place names the `blockstor snapshot resource restore` that finishes it, including for `rd clone`, whose own re-run is refused over a target that exists. | | 88 | `rg vg m --gross` / `rg vg m` with `flags` | MISSING_FEATURE | permanent | Upstream's one volume-group flag is `GROSS_SIZE`, which sizes a spawned volume gross. Nothing in blockstor sizes a spawned volume gross, so the volume-group modify door answers 501 to any `flags` in the body and writes nothing, the props half included, rather than a 200 for a request that changes nothing. | | 89 | `delete_namespaces` on every props-modify door | BEHAVIOR | permanent | Not a divergence; recorded so it is not reopened. As upstream's `PropsContainer.removeNamespace` does, a namespace removes the keys below it and leaves a key spelled exactly as the namespace: `DrbdOptions` takes `DrbdOptions/Net/protocol` and keeps `DrbdOptions`. Trailing slashes name the same namespace. Every door and every guard in front of one read it through the same helper. | diff --git a/internal/cli/definition.go b/internal/cli/definition.go index 837d6e6b..0f4b64c0 100644 --- a/internal/cli/definition.go +++ b/internal/cli/definition.go @@ -22,6 +22,7 @@ import ( "context" "errors" "fmt" + "maps" "slices" "sort" "strings" @@ -29,6 +30,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" "github.com/cozystack/blockstor/internal/cli/command" "github.com/cozystack/blockstor/internal/cli/view" @@ -156,7 +158,7 @@ func resourceDefinitionClone(ctx context.Context, run *runContext) error { snapName := cloneSnapshotName(target) - err = ensureCloneSnapshot(ctx, run, &src, snapName) + err = ensureCloneSnapshot(ctx, run, &src, snapName, target) if err != nil { return err } @@ -234,7 +236,9 @@ func checkCloneSnapshotIsCurrent( return nil } -func ensureCloneSnapshot(ctx context.Context, run *runContext, src *apiv1.ResourceDefinition, snapName string) error { +func ensureCloneSnapshot( + ctx context.Context, run *runContext, src *apiv1.ResourceDefinition, snapName, target string, +) error { existing, err := run.Store.Snapshots().Get(ctx, src.Name, snapName) if err == nil { return checkCloneSnapshotIsCurrent(ctx, run, src, &existing) @@ -255,6 +259,15 @@ func ensureCloneSnapshot(ctx context.Context, run *runContext, src *apiv1.Resour return err } + // Stamped as the clone's own, the way the REST door stamps it, so that + // deleting the clone may reap it; see store.CloneSnapshotOwnerProp. + snap.Props = maps.Clone(snap.Props) + if snap.Props == nil { + snap.Props = map[string]string{} + } + + snap.Props[store.CloneSnapshotOwnerProp] = target + err = run.Store.Snapshots().Create(ctx, snap) if err != nil { return fmt.Errorf("create clone snapshot %s: %w", snapName, err) diff --git a/internal/cli/rd_delete_clone_snapshot_test.go b/internal/cli/rd_delete_clone_snapshot_test.go new file mode 100644 index 00000000..529f805e --- /dev/null +++ b/internal/cli/rd_delete_clone_snapshot_test.go @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli_test + +import ( + "context" + "errors" + "strings" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// `rd clone` takes the same internal snapshot the REST door takes, and `rd d` +// of the clone reaped nothing, so the source could never be deleted through +// the CLI either. +func TestResourceDefinitionDeleteOfACloneReapsItsSnapshot(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, seedSnapshotSource) + + if got := app.Run(t.Context(), []string{"rd", "clone", "pvc-x", "pvc-reap"}); got != 0 { + t.Fatalf("clone exit = %d (stderr: %s)", got, errBuf.String()) + } + + // The owner prop is the snapshot's. Carried onto the clone, it would be + // copied into every snapshot later taken of the clone as a claim of + // ownership nobody made. + clone, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-reap") + if err != nil { + t.Fatalf("get the clone: %v", err) + } + + if owner, ok := clone.Props[store.CloneSnapshotOwnerProp]; ok { + t.Errorf("the clone carries the snapshot's owner prop %q", owner) + } + + if got := app.Run(t.Context(), []string{"rd", "d", "pvc-reap"}); got != 0 { + t.Fatalf("delete of the clone exit = %d (stderr: %s)", got, errBuf.String()) + } + + backend := appStore(t, app) + + if _, err := backend.Snapshots().Get(t.Context(), "pvc-x", "clone-pvc-reap"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the internal snapshot outlived the clone it backed: %v", err) + } + + // seedSnapshotSource also leaves an operator snapshot on pvc-x, which the + // delete has to keep refusing over. The clone's own must not be among what + // it names. + app.Run(t.Context(), []string{"rd", "d", "pvc-x"}) + + if strings.Contains(errBuf.String(), "clone-pvc-reap") { + t.Errorf("the source delete still meets the clone's snapshot: %s", errBuf.String()) + } +} + +// An operator can name a snapshot anything, `clone-` included, and +// restore it under that target. Only the prop the clone path writes makes a +// snapshot the clone's to reap. +func TestResourceDefinitionDeleteKeepsAnOperatorSnapshotNamedLikeAClone(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + _ = backend.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "clone-pvc-op", ResourceName: "pvc-x", + Nodes: []string{"node-1", "node-2"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 1 << 20}}, + }) + }) + + argv := []string{ + "s", "resource", "restore", + "--from-resource", "pvc-x", "--from-snapshot", "clone-pvc-op", "--to-resource", "pvc-op", + } + + if got := app.Run(t.Context(), argv); got != 0 { + t.Fatalf("restore exit = %d (stderr: %s)", got, errBuf.String()) + } + + if got := app.Run(t.Context(), []string{"rd", "d", "pvc-op"}); got != 0 { + t.Fatalf("delete exit = %d (stderr: %s)", got, errBuf.String()) + } + + if _, err := appStore(t, app).Snapshots().Get(t.Context(), "pvc-x", "clone-pvc-op"); err != nil { + t.Errorf("the operator's snapshot was reaped because of its name: %v", err) + } +} + +// The CLI refusal over a source still carrying a snapshot its clone left had no +// case that reached it: every fixture's list came back empty. +func TestResourceDefinitionDeleteNamesACloneSnapshotLeftBehind(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + _ = backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "pvc-left"}) + _ = backend.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "clone-pvc-gone", ResourceName: "pvc-left", + Props: map[string]string{store.CloneSnapshotOwnerProp: "pvc-gone"}, + }) + }) + + if got := app.Run(t.Context(), []string{"rd", "d", "pvc-left"}); got == 0 { + t.Fatal("delete of a source still carrying a snapshot succeeded") + } + + if !strings.Contains(errBuf.String(), "clone-pvc-gone") { + t.Errorf("refusal %q does not name the snapshot the clone left", errBuf.String()) + } +} diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 59135b9f..ef87610c 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -347,6 +347,13 @@ func snapshotRestoreResource(ctx context.Context, run *runContext) error { return fmt.Errorf("create resource definition %s: %w", def.Name, err) } + // The snapshot was read before the definition existed; see + // store.ReapClonedSnapshot. + err = store.RestoreSourceWithdrawn(ctx, run.Store, snap.ResourceName, snap.Name) + if err != nil { + return rollbackRestore(ctx, run, def.Name, planned, err) + } + // The definition is this command's own, but a concurrent run may already // have adopted it and be hydrating the same volumes. A volume that is // already there at the snapshot's size is the restore's, not a collision @@ -387,6 +394,10 @@ func newRestoredDefinition(rdName string, src *apiv1.ResourceDefinition, snap *a // the placer looks the source half up as a store key. def.Props = store.WithRestoreMarker(def.Props, snap) + // The owner prop belongs to the snapshot; see store.CloneSnapshotOwnerProp. + delete(def.Props, store.CloneSnapshotOwnerProp) + delete(def.Props, store.CloneSnapshotReapingProp) + return def } @@ -469,6 +480,21 @@ func restoreIntoExistingTarget( return finishConcurrentRestore(ctx, run, args, existing.Name, snap, createErr) } + return placeOverMarkedTarget(ctx, run, existing.Name, snap, planned) +} + +// placeOverMarkedTarget places the planned replicas on a target that carries +// this restore's marker. The snapshot was read before the marker went on, or +// before this run found it there, so it is read back first; see +// store.ReapClonedSnapshot. +func placeOverMarkedTarget( + ctx context.Context, run *runContext, rdName string, snap *apiv1.Snapshot, planned []apiv1.Resource, +) error { + err := store.RestoreSourceWithdrawn(ctx, run.Store, snap.ResourceName, snap.Name) + if err != nil { + return fmt.Errorf("restore into %s: %w", rdName, err) + } + return placeOverExisting(ctx, run, planned) } @@ -630,7 +656,7 @@ func finishRestoreLeftover( return err } - return placeOverExisting(ctx, run, planned) + return placeOverMarkedTarget(ctx, run, rdName, snap, planned) } // answerFinishedLeftover reports a finished leftover of this restore as diff --git a/internal/cli/snapshot_restore_withdraw_test.go b/internal/cli/snapshot_restore_withdraw_test.go new file mode 100644 index 00000000..d48aa5c1 --- /dev/null +++ b/internal/cli/snapshot_restore_withdraw_test.go @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli_test + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// The CLI restore reads the snapshot back after it creates the definition, +// the same way the REST door does, and withdraws from one a reap has marked. +func TestSnapshotRestoreWithdrawsFromASnapshotBeingReaped(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + _ = backend.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "clone-pvc-old", ResourceName: "pvc-x", + Nodes: []string{"node-1", "node-2"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 1 << 20}}, + Props: map[string]string{ + store.CloneSnapshotOwnerProp: "pvc-old", + store.CloneSnapshotReapingProp: "pvc-old@" + time.Now().UTC().Format(time.RFC3339), + }, + }) + }) + + if got := app.Run(t.Context(), []string{ + "s", "resource", "restore", "--from-resource", "pvc-x", + "--from-snapshot", "clone-pvc-old", "--to-resource", "pvc-third", + }); got == 0 { + t.Fatal("restore from a snapshot being reaped succeeded") + } + + if !strings.Contains(errBuf.String(), "being deleted") { + t.Errorf("refusal %q does not say the snapshot is going", errBuf.String()) + } + + if _, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-third"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the definition restored from a snapshot being reaped was left behind: %v", err) + } +} + +// The CLI restore drops the snapshot's reaping mark the way the REST door does. +func TestSnapshotRestoreDropsAnExpiredReapingMark(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + _ = backend.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "snap-stale", ResourceName: "pvc-x", + Nodes: []string{"node-1", "node-2"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 1 << 20}}, + Props: map[string]string{ + store.CloneSnapshotReapingProp: "gone@" + time.Now().Add(-time.Hour).UTC().Format(time.RFC3339), + }, + }) + }) + + if got := app.Run(t.Context(), []string{ + "s", "resource", "restore", "--from-resource", "pvc-x", + "--from-snapshot", "snap-stale", "--to-resource", "pvc-stale", + }); got != 0 { + t.Fatalf("restore over an expired mark exit = %d (stderr: %s)", got, errBuf.String()) + } + + rd, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-stale") + if err != nil { + t.Fatalf("read the restored definition: %v", err) + } + + if mark, ok := rd.Props[store.CloneSnapshotReapingProp]; ok { + t.Errorf("the restored definition carries the snapshot's reaping mark %q", mark) + } +} diff --git a/internal/cli/write.go b/internal/cli/write.go index 35ec6a18..b8e49760 100644 --- a/internal/cli/write.go +++ b/internal/cli/write.go @@ -340,10 +340,27 @@ func resourceDefinitionDelete(ctx context.Context, run *runContext) error { } if len(snaps) > 0 { + // A snapshot a clone took for itself outlives the clone when its + // reap was skipped, failed or kept it for a restore; nothing else + // names it now. + left, leftErr := store.CloneSnapshotsLeftBehind(ctx, run.Store, snaps) + if leftErr != nil { + _, _ = fmt.Fprintf(run.Err, "warning: could not tell which snapshots clones left behind: %v\n", leftErr) + } else if !left.Empty() { + cause, correction := left.Explain(name) + + return fmt.Errorf("%w: %s has %d snapshot(s): %s; %s", + errDefinitionHasSnapshots, name, len(snaps), cause, correction) + } + return fmt.Errorf("%w: %s has %d snapshot(s); delete them first", errDefinitionHasSnapshots, name, len(snaps)) } + // Read before the delete: the definition's own props are the only record + // of the internal snapshot a clone took on its source. + clonedFrom := store.OwnedCloneSnapshot(ctx, run.Store, name) + // Then the replicas. Dropping the definition alone leaves them with no // parent to stamp a deletion on, so the satellite finalizer never runs // and the DRBD minor, port and peer entries stay live on every node. @@ -357,6 +374,15 @@ func resourceDefinitionDelete(ctx context.Context, run *runContext) error { return fmt.Errorf("delete resource definition %s: %w", name, err) } + // The internal snapshot the clone took on its source goes with it, the + // way the REST door reaps it, or the source could never be deleted again. + // Not fatal: the definition is gone, and a kept snapshot is named by the + // refusal a later delete of the source meets. + reapErr := store.ReapClonedSnapshot(ctx, run.Store, clonedFrom) + if reapErr != nil { + _, _ = fmt.Fprintf(run.Err, "warning: internal clone snapshot kept: %v\n", reapErr) + } + return nil } diff --git a/pkg/api/v1/snapshot.go b/pkg/api/v1/snapshot.go index fe59b651..6815dc9c 100644 --- a/pkg/api/v1/snapshot.go +++ b/pkg/api/v1/snapshot.go @@ -129,4 +129,7 @@ const ( SnapshotFlagFailedDeployment = "FAILED_DEPLOYMENT" SnapshotFlagFailedDisconnect = "FAILED_DISCONNECT" SnapshotFlagSuccessful = "SUCCESSFUL" + // SnapshotFlagDelete marks a snapshot whose delete was issued and has + // not finished: a finalizer still holds it. + SnapshotFlagDelete = "DELETE" ) diff --git a/pkg/rest/cache_invalidation_bug_124_test.go b/pkg/rest/cache_invalidation_bug_124_test.go index 77f6b066..ef2b08f5 100644 --- a/pkg/rest/cache_invalidation_bug_124_test.go +++ b/pkg/rest/cache_invalidation_bug_124_test.go @@ -151,6 +151,10 @@ func (l *laggingRDs) List(ctx context.Context) ([]apiv1.ResourceDefinition, erro return l.inner.List(ctx) //nolint:wrapcheck // test helper } +func (l *laggingRDs) ListUncached(ctx context.Context) ([]apiv1.ResourceDefinition, error) { + return l.inner.ListUncached(ctx) //nolint:wrapcheck // test helper +} + func (l *laggingRDs) Get(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { return l.inner.Get(ctx, name) //nolint:wrapcheck // test helper } diff --git a/pkg/rest/rd_clone.go b/pkg/rest/rd_clone.go index f85d78c7..5591fbc5 100644 --- a/pkg/rest/rd_clone.go +++ b/pkg/rest/rd_clone.go @@ -1474,6 +1474,8 @@ func (s *Server) ensureCloneSnapshot( return nil, false, false } + stampCloneSnapshotOwner(&snap, cloneName) + snap.Snapshots = makeSnapshotPerNode(snapName, snap.Nodes, snap.VolumeDefinitions) err = s.Store.Snapshots().Create(ctx, &snap) @@ -1490,6 +1492,20 @@ func (s *Server) ensureCloneSnapshot( return &snap, false, true } +// stampCloneSnapshotOwner marks the snapshot as the clone's own, so deleting +// the clone may reap it. A name cannot carry that: an operator can call a +// snapshot `clone-` and restore it under that target, and the marker a +// restore writes is the one a clone writes. The map is copied first, since the +// hydration hands over the source definition's own. +func stampCloneSnapshotOwner(snap *apiv1.Snapshot, cloneName string) { + snap.Props = maps.Clone(snap.Props) + if snap.Props == nil { + snap.Props = map[string]string{} + } + + snap.Props[store.CloneSnapshotOwnerProp] = cloneName +} + // cloneSourceIsNotBeingDeleted mirrors the snapshot-create Bug 180 gate: a // source RD mid-tear-down would reap the internal snapshot and the clone // marker from under the satellite's restore. diff --git a/pkg/rest/rd_delete_clone_snapshot_test.go b/pkg/rest/rd_delete_clone_snapshot_test.go new file mode 100644 index 00000000..be168eca --- /dev/null +++ b/pkg/rest/rd_delete_clone_snapshot_test.go @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "encoding/json" + "net/http" + "strings" + "testing" + + "github.com/cockroachdb/errors" + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +func deleteRD(t *testing.T, base, name string) int { + t.Helper() + + resp := httpDelete(t, base+"/v1/resource-definitions/"+name) + _ = resp.Body.Close() + + return resp.StatusCode +} + +// The clone takes an internal snapshot on the SOURCE, and nothing reaped it: +// the target could be deleted and the snapshot stayed, so the source could +// never be deleted again through the API, which refuses a definition that has +// snapshots. Every clone from CSI goes through this path. +func TestRDDeleteReapsTheInternalCloneSnapshotFromTheSource(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-reap") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-reap", "dst-reap", nil); code != http.StatusCreated { + t.Fatalf("clone = %d, want 201", code) + } + + if _, err := st.Snapshots().Get(ctx, "src-reap", cloneSnapshotName("dst-reap")); err != nil { + t.Fatalf("fixture: the clone took no internal snapshot: %v", err) + } + + if clone, err := st.ResourceDefinitions().Get(ctx, "dst-reap"); err != nil { + t.Fatalf("read the clone: %v", err) + } else if owner, ok := clone.Props[store.CloneSnapshotOwnerProp]; ok { + t.Errorf("the clone carries the snapshot's owner prop %q", owner) + } + + if code := deleteRD(t, base, "dst-reap"); code != http.StatusOK { + t.Fatalf("delete of the clone = %d, want 200", code) + } + + if _, err := st.Snapshots().Get(ctx, "src-reap", cloneSnapshotName("dst-reap")); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the internal snapshot outlived the clone it backed: %v", err) + } + + if code := deleteRD(t, base, "src-reap"); code != http.StatusOK { + t.Errorf("delete of the source = %d, want 200: the source is undeletable while that snapshot stands", code) + } +} + +// A restore's marker names an operator's snapshot, which is somebody's data. +// Only the snapshot a clone derived from its own target name is reaped. +func TestRDDeleteLeavesTheSnapshotARestoreCameFrom(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-keep") + seedRestoreSnapshot(t, st, "src-keep", "snap-keep", []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := restoreOnce(t, base, "src-keep", "snap-keep", + map[string]any{"to_resource": "dst-keep"}); code != http.StatusCreated { + t.Fatalf("restore = %d, want 201", code) + } + + if code := deleteRD(t, base, "dst-keep"); code != http.StatusOK { + t.Fatalf("delete of the restored definition = %d, want 200", code) + } + + if _, err := st.Snapshots().Get(ctx, "src-keep", "snap-keep"); err != nil { + t.Errorf("the operator's snapshot was reaped with the restore that used it: %v", err) + } +} + +// A restore writes the same marker a clone does, and an operator can name a +// snapshot `clone-` and restore it under that target. Ownership read +// off the name destroyed that snapshot with the definition. +func TestRDDeleteKeepsAnOperatorSnapshotNamedLikeTheClonesOwn(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-op") + seedRestoreSnapshot(t, st, "src-op", cloneSnapshotName("dst-op"), []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := restoreOnce(t, base, "src-op", cloneSnapshotName("dst-op"), + map[string]any{"to_resource": "dst-op"}); code != http.StatusCreated { + t.Fatalf("restore = %d, want 201", code) + } + + if code := deleteRD(t, base, "dst-op"); code != http.StatusOK { + t.Fatalf("delete = %d, want 200", code) + } + + if _, err := st.Snapshots().Get(ctx, "src-op", cloneSnapshotName("dst-op")); err != nil { + t.Errorf("the operator's snapshot was reaped because of its name: %v", err) + } +} + +// The internal snapshot is visible in `s l`, so a third definition can be +// restored from it, and that definition keeps reading it through its marker. +func TestRDDeleteKeepsTheCloneSnapshotAnotherDefinitionWasRestoredFrom(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-dep") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-dep", "dst-dep", nil); code != http.StatusCreated { + t.Fatalf("clone = %d, want 201", code) + } + + if code := restoreOnce(t, base, "src-dep", cloneSnapshotName("dst-dep"), + map[string]any{"to_resource": "third-dep"}); code != http.StatusCreated { + t.Fatalf("restore from the internal snapshot = %d, want 201", code) + } + + if code := deleteRD(t, base, "dst-dep"); code != http.StatusOK { + t.Fatalf("delete of the clone = %d, want 200", code) + } + + if _, err := st.Snapshots().Get(ctx, "src-dep", cloneSnapshotName("dst-dep")); err != nil { + t.Errorf("the snapshot third-dep was restored from was reaped with the clone: %v", err) + } +} + +// A reap that was skipped or failed cannot be re-run by deleting the clone +// again, since it is gone. The refusal a delete of the source meets is the one +// place that still sees the snapshot, so it names it. +func TestRDDeleteRefusalNamesAnInternalSnapshotItsCloneLeftBehind(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-orph") + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: cloneSnapshotName("gone-orph"), + ResourceName: "src-orph", + Nodes: []string{"node-a"}, + Props: map[string]string{store.CloneSnapshotOwnerProp: "gone-orph"}, + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 64 * 1024}}, + }); err != nil { + t.Fatalf("seed the orphaned snapshot: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := httpDelete(t, base+"/v1/resource-definitions/src-orph") + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusConflict { + t.Fatalf("delete of the source = %d, want 409", resp.StatusCode) + } + + var rcs []apiv1.APICallRc + if err := json.NewDecoder(resp.Body).Decode(&rcs); err != nil || len(rcs) == 0 { + t.Fatalf("decode the refusal: %v", err) + } + + if !strings.Contains(rcs[0].Cause, cloneSnapshotName("gone-orph")) { + t.Errorf("refusal cause %q does not name the snapshot the clone left", rcs[0].Cause) + } +} + +// A snapshot the reap kept because another definition was restored from it is +// not deletable. The refusal on the source used to call it an orphan and tell +// the operator to delete it, which takes the point-in-time from under the +// definition still restoring from it. +func TestRDDeleteRefusalNamesTheDefinitionThatKeepsACloneSnapshot(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-use") + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-use", "dst-use", nil); code != http.StatusCreated { + t.Fatalf("clone = %d, want 201", code) + } + + if code := restoreOnce(t, base, "src-use", cloneSnapshotName("dst-use"), + map[string]any{"to_resource": "third-use"}); code != http.StatusCreated { + t.Fatalf("restore from the internal snapshot = %d, want 201", code) + } + + if code := deleteRD(t, base, "dst-use"); code != http.StatusOK { + t.Fatalf("delete of the clone = %d, want 200", code) + } + + resp := httpDelete(t, base+"/v1/resource-definitions/src-use") + defer func() { _ = resp.Body.Close() }() + + var rcs []apiv1.APICallRc + if err := json.NewDecoder(resp.Body).Decode(&rcs); err != nil || len(rcs) == 0 { + t.Fatalf("decode the refusal (status %d): %v", resp.StatusCode, err) + } + + rc := rcs[0] + + if !strings.Contains(rc.Correc, "keep "+cloneSnapshotName("dst-use")+" while third-use exists") { + t.Errorf("correction %q does not name the definition restoring from the snapshot", rc.Correc) + } + + // The snapshot delete door does not check for dependents, so a command + // next to "kept because" is one an operator runs. + if strings.Contains(rc.Correc, "s d src-use "+cloneSnapshotName("dst-use")) { + t.Errorf("correction %q tells the operator to delete a snapshot still in use", rc.Correc) + } +} + +// A snapshot a version before the owner prop took is never reaped, and the +// operator was never told which one keeps the source refused. +func TestRDDeleteRefusalNamesAnUnownedCloneSnapshot(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-old") + seedRestoreSnapshot(t, st, "src-old", cloneSnapshotName("gone-old"), []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + resp := httpDelete(t, base+"/v1/resource-definitions/src-old") + defer func() { _ = resp.Body.Close() }() + + var rcs []apiv1.APICallRc + if err := json.NewDecoder(resp.Body).Decode(&rcs); err != nil || len(rcs) == 0 { + t.Fatalf("decode the refusal (status %d): %v", resp.StatusCode, err) + } + + if !strings.Contains(rcs[0].Cause, cloneSnapshotName("gone-old")) { + t.Errorf("cause %q does not name the unowned clone snapshot", rcs[0].Cause) + } +} + +func refusalOf(t *testing.T, base, rdName string) apiv1.APICallRc { + t.Helper() + + resp := httpDelete(t, base+"/v1/resource-definitions/"+rdName) + defer func() { _ = resp.Body.Close() }() + + var rcs []apiv1.APICallRc + if err := json.NewDecoder(resp.Body).Decode(&rcs); err != nil || len(rcs) == 0 { + t.Fatalf("decode the refusal (status %d): %v", resp.StatusCode, err) + } + + return rcs[0] +} + +// The bucket decides what the operator is told to run: a stamped orphan gets +// the command outright, one never stamped gets it behind a warning. Both carry +// the snapshot's own name, since a placeholder pasted into a shell is a +// redirect and `s d` of a name that does not exist exits 0. +func TestRDDeleteRefusalWordsEachLeftSnapshotByItsBucket(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-bkt") + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "clone-owned", ResourceName: "src-bkt", + Props: map[string]string{store.CloneSnapshotOwnerProp: "owned"}, + }); err != nil { + t.Fatalf("seed the stamped orphan: %v", err) + } + + seedRestoreSnapshot(t, st, "src-bkt", "clone-legacy", []string{"node-a"}) + + base, stop := startServerWithStore(t, st) + defer stop() + + rc := refusalOf(t, base, "src-bkt") + + if !strings.Contains(rc.Correc, "`linstor s d src-bkt clone-owned`; ") || + strings.Contains(rc.Correc, "depends on clone-owned") { + t.Errorf("correction %q does not give the stamped orphan its command outright", rc.Correc) + } + + if !strings.Contains(rc.Correc, + "if nothing of yours depends on clone-legacy, `linstor s d src-bkt clone-legacy`") { + t.Errorf("correction %q does not put the unstamped snapshot behind a warning", rc.Correc) + } + + if !strings.HasSuffix(rc.Correc, "then delete src-bkt again") { + t.Errorf("correction %q does not end on the source delete", rc.Correc) + } +} diff --git a/pkg/rest/resource_definitions.go b/pkg/rest/resource_definitions.go index 3e893dc4..77bd6980 100644 --- a/pkg/rest/resource_definitions.go +++ b/pkg/rest/resource_definitions.go @@ -1207,24 +1207,14 @@ func (s *Server) handleRDDelete(w http.ResponseWriter, r *http.Request) { // stamps DeletionTimestamp on every replica, a failed RD-delete // leaves the cluster half-torn-down (children gone, parent // kept, snapshots orphaned) which no retry can reconcile. - snaps, err := s.Store.Snapshots().ListByDefinitionUncached(r.Context(), name) - if err != nil && !errors.Is(err, store.ErrNotFound) { - writeStoreError(w, err) - + if !s.rdHasNoSnapshots(w, r, name) { return } - if len(snaps) > 0 { - writeJSON(w, http.StatusConflict, []apiv1.APICallRc{{ - RetCode: apiCallRcError | apiCallRcFailExistsSnapshotDfn, - Message: "Cannot delete resource definition '" + name + "' because it has snapshots.", - ObjRefs: map[string]string{ - objRefRscDfn: name, - }, - }}) - - return - } + // Read before the delete, used after it: the definition's own props + // are the only record of the internal snapshot a clone left on its + // source. + clonedFrom := store.OwnedCloneSnapshot(r.Context(), s.Store, name) // Cascade the delete to all child Resource replicas BEFORE // dropping the RD itself. Without this, child Resources are @@ -1243,7 +1233,7 @@ func (s *Server) handleRDDelete(w http.ResponseWriter, r *http.Request) { // existing `blockstor.cozystack.io/satellite-resource` // finalizer then drains DRBD before the apiserver removes // the object. - err = s.cascadeDeleteResources(r.Context(), name) + err := s.cascadeDeleteResources(r.Context(), name) if err != nil { writeStoreError(w, err) @@ -1289,6 +1279,14 @@ func (s *Server) handleRDDelete(w http.ResponseWriter, r *http.Request) { // close: either ordering yields a clean cluster. s.sweepOrphanSnapshotsAfterRDDelete(r.Context(), name) + // And the internal snapshot the clone of this definition took on its + // SOURCE. Nothing else reaps it: the operator-facing snapshot doors + // and the auto-snapshot reaper are all driven by a name or a label + // this one never carries, so it outlived the target it was taken for + // and left the source undeletable through this very handler, which + // refuses a definition that has snapshots. + reapClonedSnapshot(r.Context(), s.Store, clonedFrom) + // Bug 124: block the response until the local informer cache has // observed the RD + child Resource deletions. Without this gate, // `linstor rd d ` returns SUCCESS and the very next @@ -1302,6 +1300,62 @@ func (s *Server) handleRDDelete(w http.ResponseWriter, r *http.Request) { }}) } +// rdHasNoSnapshots is handleRDDelete's pre-walk: it refuses, and answers +// false, while any Snapshot still hangs off the definition. +func (s *Server) rdHasNoSnapshots(w http.ResponseWriter, r *http.Request, name string) bool { + // Uncached: a snapshot that raced the delete and has not reached the + // informer is exactly the one this refusal exists for. + snaps, err := s.Store.Snapshots().ListByDefinitionUncached(r.Context(), name) + if err != nil && !errors.Is(err, store.ErrNotFound) { + writeStoreError(w, err) + + return false + } + + if len(snaps) > 0 { + refusal := apiv1.APICallRc{ + RetCode: apiCallRcError | apiCallRcFailExistsSnapshotDfn, + Message: "Cannot delete resource definition '" + name + "' because it has snapshots.", + ObjRefs: map[string]string{ + objRefRscDfn: name, + }, + } + + // A snapshot a clone took for itself outlives the clone when its + // reap was skipped or failed, and a repeated delete of the clone + // cannot re-run it: the clone is already gone. This refusal is the + // one place that still sees it, so it names it. + left, leftErr := store.CloneSnapshotsLeftBehind(r.Context(), s.Store, snaps) + if leftErr != nil { + log.FromContext(r.Context()).V(1).Info("could not tell which snapshots clones left behind", + "resourceDefinition", name, "error", leftErr.Error()) + } else if !left.Empty() { + refusal.Cause, refusal.Correc = left.Explain(name) + } + + writeJSON(w, http.StatusConflict, []apiv1.APICallRc{refusal}) + + return false + } + + return true +} + +// reapClonedSnapshot drops the internal snapshot the deleted definition was +// cloned from, when the clone path stamped it as that definition's own; see +// store.ReapClonedSnapshot. +// +// Best-effort, like the sweep below: the delete has already succeeded and the +// operator has been told so. A snapshot kept because another definition was +// restored from it, or one whose delete failed, stays visible in `linstor s l`, +// and the refusal a later delete of the source meets names it. +func reapClonedSnapshot(ctx context.Context, st store.Store, ref store.ClonedSnapshotRef) { + err := store.ReapClonedSnapshot(ctx, st, ref) + if err != nil { + log.FromContext(ctx).V(1).Info("internal clone snapshot kept", "reason", err.Error()) + } +} + // sweepOrphanSnapshotsAfterRDDelete drops any Snapshot rows that // raced past the pre-Delete `ensureNoSnapDfns` walk in handleRDDelete. // Bug 180: a concurrent `linstor s c ` can slip between diff --git a/pkg/rest/snapshot_restore.go b/pkg/rest/snapshot_restore.go index 793e4f76..09c99f2f 100644 --- a/pkg/rest/snapshot_restore.go +++ b/pkg/rest/snapshot_restore.go @@ -1364,6 +1364,12 @@ func (s *Server) materializeRestoredRD(ctx context.Context, srcRD string, req *s // every replica of the new RD clones from the same source. newRD.Props = store.WithRestoreMarker(newRD.Props, snap) + // The owner prop is the snapshot's, not the definition's: copied onward, + // every snapshot later taken of this definition would inherit a claim of + // ownership it was never given. + delete(newRD.Props, store.CloneSnapshotOwnerProp) + delete(newRD.Props, store.CloneSnapshotReapingProp) + created, adoptedRG, err := s.createOrAdoptRestoredRD(ctx, &newRD, snap, overrides) if err != nil { return materialisedRD{}, err @@ -1377,6 +1383,16 @@ func (s *Server) materializeRestoredRD(ctx context.Context, srcRD string, req *s made = createdRD(newRD.Name, newRD.ResourceGroupName) } + // The snapshot was read before the definition existed, and a reap that + // listed dependents in between could not see this one. Read it back now + // the definition is there; see store.ReapClonedSnapshot. The withdraw has + // already taken back what this request created, so its error is not one + // to roll back over. + err = store.RestoreSourceWithdrawn(ctx, s.Store, snap.ResourceName, snap.Name) + if err != nil { + return materialisedRD{}, s.withdrawRestoredRD(ctx, newRD.Name, created, err) + } + err = hydrateVolumesFromSnapshot(ctx, s, newRD.Name, snap, true) if err != nil { return made, made.failedAfterWrite(err) @@ -1475,6 +1491,35 @@ func (s *Server) claimAdoptedLeftover(ctx context.Context, rdName string, snap * return err //nolint:wrapcheck // store names the definition } +// withdrawRestoredRD takes back a definition this request created from a +// snapshot that went away under it, before anything was hydrated into it, and +// answers as for a snapshot that does not exist. +// +// A definition an earlier attempt left is not this request's to delete: it +// may hold what that attempt hydrated. It cannot be finished without the +// snapshot either, so the answer names it and the way out, rather than a 404 +// every retry repeats with nothing pointing at the leftover. +func (s *Server) withdrawRestoredRD(ctx context.Context, rdName string, created bool, cause error) error { + if created { + err := s.Store.ResourceDefinitions().Delete(ctx, rdName) + if err != nil && !errors.Is(err, store.ErrNotFound) { + log.FromContext(ctx).Info("could not withdraw a definition whose snapshot went away", + "resourceDefinition", rdName, "error", err.Error()) + } + } + + if !errors.Is(cause, store.ErrRestoreSourceWithdrawn) { + return cause + } + + if created { + return errors.Wrap(store.ErrNotFound, cause.Error()) + } + + return errors.Wrapf(store.ErrNotFound, "%s; '%s', left by an earlier attempt at this restore, "+ + "cannot be finished without it: delete it with `linstor rd d %s`", cause.Error(), rdName, rdName) +} + // materialisedRD is what a materialisation wrote, and whether the definition // under that name is this call's to undo. // diff --git a/pkg/rest/snapshot_restore_withdraw_test.go b/pkg/rest/snapshot_restore_withdraw_test.go new file mode 100644 index 00000000..47a6327a --- /dev/null +++ b/pkg/rest/snapshot_restore_withdraw_test.go @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rest + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "testing" + "time" + + "github.com/cockroachdb/errors" + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// snapshotGoesOnCreate deletes one snapshot just before a definition is +// created: the reap of a clone landing between the restore's snapshot read +// and its definition create. +type snapshotGoesOnCreate struct { + store.Store + + source, snapshot string +} + +type snapshotGoesOnCreateRDs struct { + store.ResourceDefinitionStore + + outer snapshotGoesOnCreate +} + +func (s snapshotGoesOnCreate) ResourceDefinitions() store.ResourceDefinitionStore { + return snapshotGoesOnCreateRDs{ResourceDefinitionStore: s.Store.ResourceDefinitions(), outer: s} +} + +func (r snapshotGoesOnCreateRDs) Create(ctx context.Context, rd *apiv1.ResourceDefinition) error { + _ = r.outer.Store.Snapshots().Delete(ctx, r.outer.source, r.outer.snapshot) + + return r.ResourceDefinitionStore.Create(ctx, rd) //nolint:wrapcheck // pass-through test double +} + +// A reap whose dependent list was taken before the restore created its +// definition deleted the snapshot from under it, and the restore answered 201 +// over a definition the satellite can only bring up blank. +func TestSnapshotRestoreWithdrawsWhenItsSnapshotGoesBeforeTheDefinitionLands(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, backend, "src-gone") + seedRestoreSnapshot(t, backend, "src-gone", "snap-gone", []string{"node-a"}) + + base, stop := startServerWithStore(t, snapshotGoesOnCreate{ + Store: backend, source: "src-gone", snapshot: "snap-gone", + }) + defer stop() + + code := restoreOnce(t, base, "src-gone", "snap-gone", map[string]any{"to_resource": "dst-gone"}) + if code != http.StatusNotFound { + t.Errorf("restore over a snapshot deleted under it = %d, want 404", code) + } + + if _, err := backend.ResourceDefinitions().Get(ctx, "dst-gone"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the definition restored from a deleted snapshot was left behind: %v", err) + } +} + +// The reap marks before it lists, so a restore whose create landed after the +// list reads the mark back and withdraws. +func TestSnapshotRestoreWithdrawsFromASnapshotBeingReaped(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-reap") + + seedRestoreSnapshot(t, st, "src-reap", "clone-dst-reap", []string{"node-a"}) + + snap, err := st.Snapshots().Get(ctx, "src-reap", "clone-dst-reap") + if err != nil { + t.Fatalf("read the seeded snapshot: %v", err) + } + + snap.Props = map[string]string{ + store.CloneSnapshotOwnerProp: "dst-reap", + store.CloneSnapshotReapingProp: "dst-reap@" + time.Now().UTC().Format(time.RFC3339), + } + + if err := st.Snapshots().Update(ctx, &snap); err != nil { + t.Fatalf("mark the snapshot as being reaped: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + code := restoreOnce(t, base, "src-reap", "clone-dst-reap", map[string]any{"to_resource": "third-reap"}) + if code != http.StatusNotFound { + t.Errorf("restore from a snapshot being reaped = %d, want 404", code) + } + + if _, err := st.ResourceDefinitions().Get(ctx, "third-reap"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the definition restored from a snapshot being reaped was left behind: %v", err) + } +} + +// A mark no live reap owns is not withdrawn on, and it is the snapshot's: a +// definition restored from the snapshot does not carry it onward to every +// snapshot later taken of it. +func TestSnapshotRestoreDropsAnExpiredReapingMark(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-stale") + seedRestoreSnapshot(t, st, "src-stale", "snap-stale", []string{"node-a"}) + + snap, err := st.Snapshots().Get(ctx, "src-stale", "snap-stale") + if err != nil { + t.Fatalf("read the seeded snapshot: %v", err) + } + + snap.Props = map[string]string{ + store.CloneSnapshotReapingProp: "gone@" + time.Now().Add(-time.Hour).UTC().Format(time.RFC3339), + } + + if err := st.Snapshots().Update(ctx, &snap); err != nil { + t.Fatalf("leave an expired mark: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := restoreOnce(t, base, "src-stale", "snap-stale", + map[string]any{"to_resource": "dst-stale"}); code != http.StatusCreated { + t.Fatalf("restore over an expired mark = %d, want 201", code) + } + + rd, err := st.ResourceDefinitions().Get(ctx, "dst-stale") + if err != nil { + t.Fatalf("read the restored definition: %v", err) + } + + if mark, ok := rd.Props[store.CloneSnapshotReapingProp]; ok { + t.Errorf("the restored definition carries the snapshot's reaping mark %q", mark) + } +} + +// A leftover an earlier attempt left is not this request's to delete, and the +// answer names it: without that every retry repeats a 404 about the snapshot +// with nothing pointing at the definition that can no longer be finished. +func TestSnapshotRestoreKeepsAnEarlierLeftoverWhenItsSnapshotGoes(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + ctx := t.Context() + seedDeployedCloneSource(t, backend, "src-left") + seedRestoreSnapshot(t, backend, "src-left", "snap-left", []string{"node-a"}) + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst-left", + Props: map[string]string{restoreFromSnapshotKey: restoreMarker("src-left", "snap-left")}, + }); err != nil { + t.Fatalf("seed the leftover: %v", err) + } + + base, stop := startServerWithStore(t, snapshotGoesOnCreate{ + Store: backend, source: "src-left", snapshot: "snap-left", + }) + defer stop() + + raw, err := json.Marshal(map[string]any{"to_resource": "dst-left"}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + resp := httpPost(t, base+"/v1/resource-definitions/src-left/snapshot-restore-resource/snap-left", raw) + defer func() { _ = resp.Body.Close() }() + + var rcs []apiv1.APICallRc + if err := json.NewDecoder(resp.Body).Decode(&rcs); err != nil || len(rcs) == 0 { + t.Fatalf("decode the answer (status %d): %v", resp.StatusCode, err) + } + + if resp.StatusCode != http.StatusNotFound { + t.Errorf("restore over a leftover whose snapshot went = %d, want 404", resp.StatusCode) + } + + if !strings.Contains(rcs[0].Message, "linstor rd d dst-left") { + t.Errorf("answer %q does not name the leftover", rcs[0].Message) + } + + if _, err := backend.ResourceDefinitions().Get(ctx, "dst-left"); err != nil { + t.Errorf("the earlier attempt's leftover was deleted: %v", err) + } +} diff --git a/pkg/rest/snapshots.go b/pkg/rest/snapshots.go index f6f48715..3fb9e4b1 100644 --- a/pkg/rest/snapshots.go +++ b/pkg/rest/snapshots.go @@ -396,9 +396,13 @@ const legacySnapshotFlagFailed = "FAILED" // isTerminalSnapshotFlag reports whether a flag value already pins the // snapshot to a non-Incomplete State that outranks `SUCCESSFUL`. -// Mirrors the python CLI's elif-chain order in snapshot_cmds.show. +// Mirrors the python CLI's elif-chain order in snapshot_cmds.show, where +// DELETE comes first: a snapshot whose delete is in flight is Deleting, and +// stamping SUCCESSFUL beside it hands a client that reads the flags in any +// other order a snapshot being destroyed as a usable one. func isTerminalSnapshotFlag(flag string) bool { - return flag == apiv1.SnapshotFlagFailedDeployment || + return flag == apiv1.SnapshotFlagDelete || + flag == apiv1.SnapshotFlagFailedDeployment || flag == apiv1.SnapshotFlagFailedDisconnect || flag == apiv1.SnapshotFlagSuccessful || flag == legacySnapshotFlagFailed diff --git a/pkg/rest/snapshots_test.go b/pkg/rest/snapshots_test.go index ebaedad4..5dc45cb8 100644 --- a/pkg/rest/snapshots_test.go +++ b/pkg/rest/snapshots_test.go @@ -2417,3 +2417,43 @@ func TestSnapshotCreateScopesGateToTargetNodes_G5(t *testing.T) { t.Errorf("targeting thick n2: got %d, want 400 (thick pool refused)", badResp.StatusCode) } } + +// A snapshot whose delete is in flight is Deleting: the python CLI checks the +// DELETE flag first, and SUCCESSFUL stamped beside it reads as a usable +// snapshot to any client that checks the flags in another order. +func TestSnapshotStateNotMarkedSuccessfulWhileDeleting(t *testing.T) { + ctx := t.Context() + + st := store.NewInMemory() + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "snap-deleting"}); err != nil { + t.Fatalf("seed RD: %v", err) + } + + if err := st.Resources().Create(ctx, &apiv1.Resource{Name: "snap-deleting", NodeName: "worker-1"}); err != nil { + t.Fatalf("seed Resource: %v", err) + } + + if err := st.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "snap1", + ResourceName: "snap-deleting", + Nodes: []string{"worker-1"}, + Flags: []string{apiv1.SnapshotFlagDelete}, + Snapshots: []apiv1.SnapshotPerNode{ + {SnapshotName: "snap1", NodeName: "worker-1", CreateTimestamp: 1714000000}, + }, + }); err != nil { + t.Fatalf("seed Snapshot: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + got := decodeSnapshotPage(t, base+"/v1/view/snapshots") + if len(got) != 1 { + t.Fatalf("view len: got %d, want 1", len(got)) + } + + if slices.Contains(got[0].Flags, apiv1.SnapshotFlagSuccessful) { + t.Errorf("Flags: SUCCESSFUL stamped beside DELETE: %v", got[0].Flags) + } +} diff --git a/pkg/store/clone_snapshot.go b/pkg/store/clone_snapshot.go new file mode 100644 index 00000000..2a147433 --- /dev/null +++ b/pkg/store/clone_snapshot.go @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "context" + "errors" + "fmt" + "slices" + "sort" + "strings" + "time" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" +) + +// CloneSnapshotOwnerProp marks a snapshot the clone path took for itself. Its +// value is the name of the clone the snapshot was taken for. +// +// The marker alone cannot say whether a snapshot is the clone's own: a restore +// writes the same marker, pointing at an operator's snapshot, and an operator +// can name a snapshot anything, `clone-` included. Only a prop this +// server wrote when it took the snapshot separates the two, so a snapshot +// without it is somebody's data and is never reaped. +const CloneSnapshotOwnerProp = "Blockstor/CloneSnapshotOf" + +// ClonedSnapshotRef names the internal snapshot a clone left on its source, +// and the clone it was taken for. +type ClonedSnapshotRef struct { + Source string + Snapshot string + Clone string +} + +// OwnedCloneSnapshot answers which internal snapshot a definition was cloned +// from, read before the definition is deleted since its props are the only +// record of it. The zero value means there is none to reap: the definition is +// not a clone, the snapshot is gone, or the snapshot does not carry the owner +// prop naming this definition. +func OwnedCloneSnapshot(ctx context.Context, st Store, rdName string) ClonedSnapshotRef { + rd, err := st.ResourceDefinitions().Get(ctx, rdName) + if err != nil { + return ClonedSnapshotRef{} + } + + source, snapName, found := strings.Cut(rd.Props[RestoreFromSnapshotProp], ":") + if !found || source == "" || snapName == "" { + return ClonedSnapshotRef{} + } + + snap, err := st.Snapshots().Get(ctx, source, snapName) + if err != nil || !strings.EqualFold(snap.Props[CloneSnapshotOwnerProp], rdName) { + return ClonedSnapshotRef{} + } + + return ClonedSnapshotRef{Source: source, Snapshot: snapName, Clone: rdName} +} + +// CloneSnapshotReapingProp marks an internal clone snapshot the reap has +// started on, as `@`. A restore that created its +// definition from the snapshot reads it back past the cache and withdraws when +// it finds a live mark or no snapshot at all; see ReapClonedSnapshot for why +// both sides are needed. +const CloneSnapshotReapingProp = "Blockstor/CloneSnapshotReaping" + +// cloneSnapshotReapBudget bounds a reap from its first write to its last: no +// call the reap makes, the delete included, can land after it. +const cloneSnapshotReapBudget = 30 * time.Second + +// cloneSnapshotReapingMarkTTL is how long a reaping mark stays live. It covers +// the reap up to its delete: a mark older than this belongs to a reap that can +// no longer issue one, since cloneSnapshotReapBudget is far shorter, so it is +// ignored, and a mark left by a reap that died, or whose clean-up failed, +// cannot refuse restores from a snapshot the reap never deleted for longer +// than this. A delete that was issued is a different state and outlives any +// budget while a satellite finalizer holds the snapshot; that one is read off +// the snapshot's DELETE flag, not off the mark. The gap between the two +// budgets also absorbs clock skew between the replica that wrote the mark and +// the one that reads it. +const cloneSnapshotReapingMarkTTL = 5 * time.Minute + +// ErrCloneSnapshotInUse reports an internal clone snapshot another definition +// was restored from, which is therefore kept. +var ErrCloneSnapshotInUse = errors.New("internal clone snapshot is still a restore source") + +// ErrRestoreSourceWithdrawn reports a snapshot that was deleted, or is being +// reaped, while a definition was being restored from it. +var ErrRestoreSourceWithdrawn = errors.New("snapshot was withdrawn while a definition was restored from it") + +// ReapClonedSnapshot drops the internal snapshot once the clone it was taken +// for is gone, which is what lets the source be deleted again: both doors +// refuse to delete a definition that still has snapshots. +// +// The snapshot is visible in `s l`, so it can have been used as the source of +// a restore, and that definition keeps reading it through its own marker: the +// placer pins new replicas to the snapshot's nodes and the satellite restores +// a replica from it by name. Such a snapshot is kept, and the error says why. +// +// A dependent appears in two steps, the snapshot read and the definition +// create, so no list taken here alone can see one that is between them. The +// reap therefore marks the snapshot first, then asks the API server for +// dependents, and only then deletes; a restore creates its definition first, +// then reads the snapshot back past the cache and withdraws on a live mark or +// on a snapshot that is gone (RestoreSourceWithdrawn). Whichever of the list +// and the create comes second sees the other: a create before the list is a +// dependent the list finds, and a create after it is followed by a read that +// finds the mark. +// +// The reap runs on a context the caller cannot cancel, bounded by +// cloneSnapshotReapBudget: the delete has already been reported, and a reap +// cut between its mark and its clean-up would leave the mark behind. A mark +// that is left anyway, by a process that died or a clean-up that failed, +// expires (cloneSnapshotReapingMarkTTL). +// +// When the dependents cannot be listed the snapshot is kept and the mark +// taken off again; the refusal a later delete of the source meets names it. +func ReapClonedSnapshot(ctx context.Context, st Store, ref ClonedSnapshotRef) error { + if ref.Source == "" || ref.Snapshot == "" { + return nil + } + + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cloneSnapshotReapBudget) + defer cancel() + + err := setReapingMark(ctx, st, ref.Source, ref.Snapshot, ref.Clone+"@"+time.Now().UTC().Format(time.RFC3339)) + if errors.Is(err, ErrNotFound) { + return nil + } + + if err != nil { + return fmt.Errorf("mark internal clone snapshot %s/%s for reaping: %w", ref.Source, ref.Snapshot, err) + } + + definitions, err := st.ResourceDefinitions().ListUncached(ctx) + if err != nil { + return keepCloneSnapshot(ctx, st, ref, + fmt.Errorf("list definitions restored from %s/%s: %w", ref.Source, ref.Snapshot, err)) + } + + dependents := restoredFrom(definitions, ref.Source, ref.Snapshot) + if len(dependents) > 0 { + return keepCloneSnapshot(ctx, st, ref, fmt.Errorf("%w: %s was restored from %s:%s", + ErrCloneSnapshotInUse, dependents[0], ref.Source, ref.Snapshot)) + } + + err = st.Snapshots().Delete(ctx, ref.Source, ref.Snapshot) + if err != nil && !errors.Is(err, ErrNotFound) { + return keepCloneSnapshot(ctx, st, ref, + fmt.Errorf("delete internal clone snapshot %s/%s: %w", ref.Source, ref.Snapshot, err)) + } + + return nil +} + +// keepCloneSnapshot takes the reaping mark back off a snapshot the reap +// decided to keep, so restores from it are not refused, and returns why it +// was kept. +func keepCloneSnapshot(ctx context.Context, st Store, ref ClonedSnapshotRef, why error) error { + err := setReapingMark(ctx, st, ref.Source, ref.Snapshot, "") + if err != nil && !errors.Is(err, ErrNotFound) { + return fmt.Errorf("%w; the reaping mark stayed on it, so restores from it are refused "+ + "for up to %s: %w", why, cloneSnapshotReapingMarkTTL, err) + } + + return why +} + +// setReapingMark writes mark as the reaping mark, or removes the mark when it +// is empty. The snapshot is read past the cache: the mark this reap wrote one +// round trip earlier may not have reached the cache yet, and a removal decided +// on that read would find nothing to remove. +func setReapingMark(ctx context.Context, st Store, source, snapshot, mark string) error { + snap, err := uncachedSnapshot(ctx, st, source, snapshot) + if err != nil { + return err + } + + if mark == "" { + if _, marked := snap.Props[CloneSnapshotReapingProp]; !marked { + return nil + } + + delete(snap.Props, CloneSnapshotReapingProp) + } else { + if snap.Props == nil { + snap.Props = map[string]string{} + } + + snap.Props[CloneSnapshotReapingProp] = mark + } + + return st.Snapshots().Update(ctx, &snap) //nolint:wrapcheck // callers wrap with the snapshot they name +} + +// uncachedSnapshot reads one snapshot from the API server. +func uncachedSnapshot(ctx context.Context, st Store, source, snapshot string) (apiv1.Snapshot, error) { + snaps, err := st.Snapshots().ListByDefinitionUncached(ctx, source) + if err != nil { + return apiv1.Snapshot{}, fmt.Errorf("read snapshot %s/%s: %w", source, snapshot, err) + } + + for i := range snaps { + if strings.EqualFold(snaps[i].Name, snapshot) { + return snaps[i], nil + } + } + + return apiv1.Snapshot{}, fmt.Errorf("snapshot %s/%s: %w", source, snapshot, ErrNotFound) +} + +// reapingMarkIsLive reports whether a reaping mark belongs to a reap that can +// still delete the snapshot. A mark whose time cannot be read is not. +func reapingMarkIsLive(mark string, now time.Time) bool { + at := strings.LastIndex(mark, "@") + if at < 0 { + return false + } + + written, err := time.Parse(time.RFC3339, mark[at+1:]) + if err != nil { + return false + } + + return now.Sub(written) < cloneSnapshotReapingMarkTTL +} + +// RestoreSourceWithdrawn is the restore's half of the protocol described on +// ReapClonedSnapshot: called after the restored definition was created, it +// reads the snapshot back past the cache and reports ErrRestoreSourceWithdrawn +// when it is gone or a reap that is still live has started on it. Any other +// error means the answer could not be had, and the caller withdraws as well. +func RestoreSourceWithdrawn(ctx context.Context, st Store, source, snapshot string) error { + snap, err := uncachedSnapshot(ctx, st, source, snapshot) + if errors.Is(err, ErrNotFound) { + return fmt.Errorf("%w: %s/%s was deleted", ErrRestoreSourceWithdrawn, source, snapshot) + } + + if err != nil { + return err + } + + if slices.Contains(snap.Flags, apiv1.SnapshotFlagDelete) || + reapingMarkIsLive(snap.Props[CloneSnapshotReapingProp], time.Now()) { + return fmt.Errorf("%w: %s/%s is being deleted", ErrRestoreSourceWithdrawn, source, snapshot) + } + + return nil +} + +// restoredFrom names the definitions whose marker says they were restored +// from source:snapshot. The clone the snapshot was taken for is not skipped by +// name: by the time the reap lists, the clone is gone from the API server, as +// a definition has no finalizer to linger behind, and a definition that has +// taken its name since is a dependent like any other. +func restoredFrom(definitions []apiv1.ResourceDefinition, source, snapshot string) []string { + marker := source + ":" + snapshot + + var names []string + + for i := range definitions { + if strings.EqualFold(definitions[i].Props[RestoreFromSnapshotProp], marker) { + names = append(names, definitions[i].Name) + } + } + + return names +} + +// LeftCloneSnapshots is what the snapshots a source still carries say about +// clones that are gone. +type LeftCloneSnapshots struct { + // Deletable were taken by a clone that no longer exists, and no + // definition was restored from them. + Deletable []string + // Unowned look like a clone's internal snapshot, `clone-` for a + // definition that no longer exists, but carry no owner prop: a version + // before the prop existed took them, or an operator named one that way. + // Nothing here reaps them; they are named so the operator knows why the + // source is still refused. + Unowned []string + // InUse were taken by a clone that is gone but still have a definition + // restored from them, keyed by snapshot. + InUse map[string]string +} + +// Empty reports whether there is nothing to tell the operator. +func (l LeftCloneSnapshots) Empty() bool { + return len(l.Deletable) == 0 && len(l.Unowned) == 0 && len(l.InUse) == 0 +} + +// CloneSnapshotsLeftBehind sorts a source's snapshots that outlived the clone +// they belong to. A delete that reaped nothing, a reap that failed, and a reap +// that kept a snapshot another definition was restored from all leave one, +// and a repeated delete of the clone cannot revisit them: the clone is gone. +// The refusal a delete of the source meets is the one place that still sees +// them, so it names them, and never calls a snapshot deletable while a +// definition still restores from it. +func CloneSnapshotsLeftBehind(ctx context.Context, st Store, snaps []apiv1.Snapshot) (LeftCloneSnapshots, error) { + out := LeftCloneSnapshots{InUse: map[string]string{}} + + var definitions []apiv1.ResourceDefinition + + for i := range snaps { + owner := snaps[i].Props[CloneSnapshotOwnerProp] + legacy := owner == "" + + if legacy { + var found bool + + owner, found = strings.CutPrefix(snaps[i].Name, "clone-") + if !found || owner == "" { + continue + } + } + + _, err := st.ResourceDefinitions().Get(ctx, owner) + if !errors.Is(err, ErrNotFound) { + continue + } + + // One list answers every snapshot, taken only once one needs it. + if definitions == nil { + definitions, err = st.ResourceDefinitions().ListUncached(ctx) + if err != nil { + return LeftCloneSnapshots{}, fmt.Errorf("list definitions restored from %s: %w", snaps[i].ResourceName, err) + } + } + + dependents := restoredFrom(definitions, snaps[i].ResourceName, snaps[i].Name) + + switch { + case len(dependents) > 0: + out.InUse[snaps[i].Name] = dependents[0] + case legacy: + out.Unowned = append(out.Unowned, snaps[i].Name) + default: + out.Deletable = append(out.Deletable, snaps[i].Name) + } + } + + return out, nil +} + +// Explain words the report for the refusal both delete doors give, so the two +// cannot drift. Every command it prints names the snapshot it acts on, and the +// steps come in the order they have to run: what can go now, then what has to +// wait, then the source. A snapshot a definition still restores from gets no +// command at all: the snapshot delete door does not check for dependents, and +// a pasteable delete next to "kept because" is one an operator runs. +func (l LeftCloneSnapshots) Explain(source string) (string, string) { + size := len(l.Deletable) + len(l.Unowned) + len(l.InUse) + causes := make([]string, 0, size) + steps := make([]string, 0, size+1) + + deleteCmd := func(snapshot string) string { + return "`linstor s d " + source + " " + snapshot + "`" + } + + for _, snapshot := range l.Deletable { + causes = append(causes, "internal clone snapshot "+snapshot+ + " outlived the clone it was taken for, and nothing was restored from it") + steps = append(steps, deleteCmd(snapshot)) + } + + for _, snapshot := range l.Unowned { + causes = append(causes, "snapshot "+snapshot+" looks like the internal snapshot of a clone "+ + "that no longer exists, but was not stamped as one, so it is never reaped") + steps = append(steps, "if nothing of yours depends on "+snapshot+", "+deleteCmd(snapshot)) + } + + kept := make([]string, 0, len(l.InUse)) + for snapshot := range l.InUse { + kept = append(kept, snapshot) + } + + sort.Strings(kept) + + for _, snapshot := range kept { + causes = append(causes, "internal clone snapshot "+snapshot+" is kept because "+ + l.InUse[snapshot]+" was restored from it") + steps = append(steps, "keep "+snapshot+" while "+l.InUse[snapshot]+" exists, since "+ + l.InUse[snapshot]+" still reads it; once "+l.InUse[snapshot]+ + " is deleted, the next delete of "+source+" names it again") + } + + steps = append(steps, "then delete "+source+" again") + + return strings.Join(causes, "; "), strings.Join(steps, "; ") +} diff --git a/pkg/store/clone_snapshot_mark_test.go b/pkg/store/clone_snapshot_mark_test.go new file mode 100644 index 00000000..f8095439 --- /dev/null +++ b/pkg/store/clone_snapshot_mark_test.go @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: Apache-2.0 + +package store_test + +import ( + "context" + "errors" + "maps" + "strings" + "testing" + "time" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// cacheWithoutTheMark answers every cached snapshot read without the reaping +// mark, the way a cache that has not seen the reap's own write does. Only the +// uncached list carries it. +type cacheWithoutTheMark struct{ store.SnapshotStore } + +func withoutTheMark(snap apiv1.Snapshot) apiv1.Snapshot { + snap.Props = maps.Clone(snap.Props) + delete(snap.Props, store.CloneSnapshotReapingProp) + + return snap +} + +func (c cacheWithoutTheMark) Get(ctx context.Context, rdName, snapName string) (apiv1.Snapshot, error) { + snap, err := c.SnapshotStore.Get(ctx, rdName, snapName) + + return withoutTheMark(snap), err +} + +func (c cacheWithoutTheMark) ListByDefinition(ctx context.Context, rdName string) ([]apiv1.Snapshot, error) { + snaps, err := c.SnapshotStore.ListByDefinition(ctx, rdName) + for i := range snaps { + snaps[i] = withoutTheMark(snaps[i]) + } + + return snaps, err //nolint:wrapcheck // pass-through test double +} + +type cacheWithoutTheMarkStore struct{ store.Store } + +func (c cacheWithoutTheMarkStore) Snapshots() store.SnapshotStore { + return cacheWithoutTheMark{c.Store.Snapshots()} +} + +func seedKeptCloneSnapshot(t *testing.T, st store.Store) { + t.Helper() + + seedOwnedCloneSnapshot(t, st) + + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{ + Name: "third", + Props: map[string]string{store.RestoreFromSnapshotProp: "src:clone-dst"}, + }); err != nil { + t.Fatalf("seed the restored definition: %v", err) + } +} + +// The mark came off through a cached read, which right after the reap's own +// write may not hold it yet, and a kept snapshot then refused every restore. +func TestReapClonedSnapshotTakesTheMarkOffPastTheCache(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + seedKeptCloneSnapshot(t, backend) + + err := store.ReapClonedSnapshot(ctx, cacheWithoutTheMarkStore{backend}, + store.ClonedSnapshotRef{Source: "src", Snapshot: "clone-dst", Clone: "dst"}) + if !errors.Is(err, store.ErrCloneSnapshotInUse) { + t.Fatalf("reap = %v, want ErrCloneSnapshotInUse", err) + } + + if err := store.RestoreSourceWithdrawn(ctx, backend, "src", "clone-dst"); err != nil { + t.Errorf("a restore from the kept snapshot is refused: %v", err) + } +} + +// honoursCancel fails every call on a context that is done, which the +// in-memory store does not. +type honoursCancel struct{ store.SnapshotStore } + +func (h honoursCancel) ListByDefinitionUncached(ctx context.Context, rdName string) ([]apiv1.Snapshot, error) { + if err := ctx.Err(); err != nil { + return nil, err //nolint:wrapcheck // the context error is the point + } + + return h.SnapshotStore.ListByDefinitionUncached(ctx, rdName) //nolint:wrapcheck // pass-through test double +} + +func (h honoursCancel) Update(ctx context.Context, snap *apiv1.Snapshot) error { + if err := ctx.Err(); err != nil { + return err //nolint:wrapcheck // the context error is the point + } + + return h.SnapshotStore.Update(ctx, snap) //nolint:wrapcheck // pass-through test double +} + +type honoursCancelStore struct{ store.Store } + +func (h honoursCancelStore) Snapshots() store.SnapshotStore { + return honoursCancel{h.Store.Snapshots()} +} + +// The delete has been reported by the time the reap runs, so a caller that +// hangs up, or a SIGTERM, must not cut it between its mark and its clean-up. +func TestReapClonedSnapshotOutlivesItsCaller(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedOwnedCloneSnapshot(t, backend) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + err := store.ReapClonedSnapshot(ctx, honoursCancelStore{backend}, + store.ClonedSnapshotRef{Source: "src", Snapshot: "clone-dst", Clone: "dst"}) + if err != nil { + t.Fatalf("reap on a cancelled caller: %v", err) + } + + if _, err := backend.Snapshots().Get(t.Context(), "src", "clone-dst"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the reap stopped with its caller: %v", err) + } +} + +// A mark left by a reap that died, or whose clean-up failed, refused restores +// from a healthy snapshot for good. It expires, and one whose time cannot be +// read is not a live reap's. +func TestRestoreSourceWithdrawnIgnoresAMarkNoLiveReapCanOwn(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + mark string + withdrawn bool + }{ + {name: "fresh", mark: "dst@" + time.Now().UTC().Format(time.RFC3339), withdrawn: true}, + {name: "expired", mark: "dst@" + time.Now().Add(-time.Hour).UTC().Format(time.RFC3339)}, + {name: "unreadable", mark: "dst"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + + if err := backend.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "clone-dst", ResourceName: "src", + Props: map[string]string{store.CloneSnapshotReapingProp: tc.mark}, + }); err != nil { + t.Fatalf("seed the snapshot: %v", err) + } + + err := store.RestoreSourceWithdrawn(ctx, backend, "src", "clone-dst") + if got := errors.Is(err, store.ErrRestoreSourceWithdrawn); got != tc.withdrawn { + t.Errorf("withdrawn = %v (%v), want %v", got, err, tc.withdrawn) + } + }) + } +} + +// The steps come in the order they can run, and the source comes last. +func TestLeftCloneSnapshotsExplainOrdersTheSteps(t *testing.T) { + t.Parallel() + + _, steps := store.LeftCloneSnapshots{ + Deletable: []string{"clone-a"}, + Unowned: []string{"clone-b"}, + InUse: map[string]string{"clone-c": "third"}, + }.Explain("src") + + last := -1 + + for _, want := range []string{"s d src clone-a", "s d src clone-b", "keep clone-c", "then delete src again"} { + at := strings.Index(steps, want) + if at <= last { + t.Fatalf("steps %q: %q out of order", steps, want) + } + + last = at + } +} + +// A delete that was issued is held by the satellite finalizer until the +// on-disk delete succeeds, which outlives the reaping mark's lifetime. The +// restore reads that state off the snapshot, not off the mark's age. +func TestRestoreSourceWithdrawnOnASnapshotBeingDeleted(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + + if err := backend.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "clone-dst", ResourceName: "src", + Flags: []string{apiv1.SnapshotFlagDelete}, + Props: map[string]string{ + store.CloneSnapshotReapingProp: "dst@" + time.Now().Add(-time.Hour).UTC().Format(time.RFC3339), + }, + }); err != nil { + t.Fatalf("seed the snapshot: %v", err) + } + + err := store.RestoreSourceWithdrawn(ctx, backend, "src", "clone-dst") + if !errors.Is(err, store.ErrRestoreSourceWithdrawn) { + t.Errorf("restore from a snapshot whose delete is in flight = %v, want withdrawn", err) + } +} diff --git a/pkg/store/clone_snapshot_test.go b/pkg/store/clone_snapshot_test.go new file mode 100644 index 00000000..9eda676e --- /dev/null +++ b/pkg/store/clone_snapshot_test.go @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 + +package store_test + +import ( + "errors" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// The clone was skipped by name, on the theory that it could still be listed +// after its own delete. It cannot: the list is the API server's, and a +// definition has no finalizer to linger behind. What the skip did let through +// is a different definition that has taken the clone's name since, restored +// from the same snapshot; the reap deleted the snapshot from under it. +func TestReapClonedSnapshotCountsADefinitionThatTookTheClonesName(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + + if err := backend.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "clone-dst", ResourceName: "src", + Props: map[string]string{store.CloneSnapshotOwnerProp: "dst"}, + }); err != nil { + t.Fatalf("seed the snapshot: %v", err) + } + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "dst", + Props: map[string]string{store.RestoreFromSnapshotProp: "src:clone-dst"}, + }); err != nil { + t.Fatalf("seed the new definition under the clone's name: %v", err) + } + + err := store.ReapClonedSnapshot(ctx, backend, store.ClonedSnapshotRef{ + Source: "src", Snapshot: "clone-dst", Clone: "dst", + }) + if !errors.Is(err, store.ErrCloneSnapshotInUse) { + t.Errorf("reap = %v, want ErrCloneSnapshotInUse", err) + } + + if _, err := backend.Snapshots().Get(ctx, "src", "clone-dst"); err != nil { + t.Errorf("the snapshot the new definition restores from was reaped: %v", err) + } +} diff --git a/pkg/store/clone_snapshot_uncached_test.go b/pkg/store/clone_snapshot_uncached_test.go new file mode 100644 index 00000000..edbdb3bf --- /dev/null +++ b/pkg/store/clone_snapshot_uncached_test.go @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: Apache-2.0 + +package store_test + +import ( + "context" + "errors" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +// cacheMissesOne hides one definition from List, the way an informer that has +// not seen its create yet does, and answers ListUncached from the backend. +type cacheMissesOne struct { + store.ResourceDefinitionStore + + hidden string +} + +func (c cacheMissesOne) List(ctx context.Context) ([]apiv1.ResourceDefinition, error) { + all, err := c.ResourceDefinitionStore.List(ctx) + + out := all[:0] + for i := range all { + if all[i].Name != c.hidden { + out = append(out, all[i]) + } + } + + return out, err //nolint:wrapcheck // pass-through test double +} + +func (c cacheMissesOne) ListUncached(ctx context.Context) ([]apiv1.ResourceDefinition, error) { + return c.ResourceDefinitionStore.List(ctx) //nolint:wrapcheck // pass-through test double +} + +type cacheMissesOneStore struct { + store.Store + + hidden string +} + +func (c cacheMissesOneStore) ResourceDefinitions() store.ResourceDefinitionStore { + return cacheMissesOne{ResourceDefinitionStore: c.Store.ResourceDefinitions(), hidden: c.hidden} +} + +// Deleting the clone before the replica serving the delete has listed the +// definition restored from its snapshot took the point-in-time from under that +// definition. The question is asked of the API server. +func TestReapClonedSnapshotAsksPastTheCacheForDependents(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + + if err := backend.Snapshots().Create(ctx, &apiv1.Snapshot{ + Name: "clone-dst", ResourceName: "src", + Props: map[string]string{store.CloneSnapshotOwnerProp: "dst"}, + }); err != nil { + t.Fatalf("seed the snapshot: %v", err) + } + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "third", + Props: map[string]string{store.RestoreFromSnapshotProp: "src:clone-dst"}, + }); err != nil { + t.Fatalf("seed the restored definition: %v", err) + } + + err := store.ReapClonedSnapshot(ctx, cacheMissesOneStore{Store: backend, hidden: "third"}, + store.ClonedSnapshotRef{Source: "src", Snapshot: "clone-dst", Clone: "dst"}) + if !errors.Is(err, store.ErrCloneSnapshotInUse) { + t.Errorf("reap = %v, want ErrCloneSnapshotInUse", err) + } + + if _, err := backend.Snapshots().Get(ctx, "src", "clone-dst"); err != nil { + t.Errorf("the snapshot third restores from was reaped: %v", err) + } +} + +var errAPIServerUnavailable = errors.New("apiserver unavailable") + +// markSeenAtList records whether the snapshot carried the reaping mark when +// the reap listed dependents, and optionally fails the list. +type markSeenAtList struct { + store.ResourceDefinitionStore + + snaps store.SnapshotStore + seen *bool + listErr error +} + +func (m markSeenAtList) ListUncached(ctx context.Context) ([]apiv1.ResourceDefinition, error) { + snap, err := m.snaps.Get(ctx, "src", "clone-dst") + if err == nil { + _, *m.seen = snap.Props[store.CloneSnapshotReapingProp] + } + + if m.listErr != nil { + return nil, m.listErr + } + + return m.ResourceDefinitionStore.ListUncached(ctx) //nolint:wrapcheck // pass-through test double +} + +type markSeenAtListStore struct { + store.Store + + seen *bool + listErr error +} + +func (m markSeenAtListStore) ResourceDefinitions() store.ResourceDefinitionStore { + return markSeenAtList{ + ResourceDefinitionStore: m.Store.ResourceDefinitions(), + snaps: m.Snapshots(), seen: m.seen, listErr: m.listErr, + } +} + +func seedOwnedCloneSnapshot(t *testing.T, st store.Store) { + t.Helper() + + if err := st.Snapshots().Create(t.Context(), &apiv1.Snapshot{ + Name: "clone-dst", ResourceName: "src", + Props: map[string]string{store.CloneSnapshotOwnerProp: "dst"}, + }); err != nil { + t.Fatalf("seed the snapshot: %v", err) + } +} + +// A list taken before the mark cannot see a restore created after it, and +// nothing on the restore's side would then tell it the snapshot is going. +func TestReapClonedSnapshotMarksBeforeItLists(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedOwnedCloneSnapshot(t, backend) + + var seen bool + + err := store.ReapClonedSnapshot(t.Context(), markSeenAtListStore{Store: backend, seen: &seen}, + store.ClonedSnapshotRef{Source: "src", Snapshot: "clone-dst", Clone: "dst"}) + if err != nil { + t.Fatalf("reap: %v", err) + } + + if !seen { + t.Error("the reap listed dependents before it marked the snapshot") + } +} + +// A snapshot the reap keeps must take restores again, or the one it kept for +// a dependent is refused to the next. +func TestReapClonedSnapshotTakesTheMarkOffWhatItKeeps(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + listErr error + want error + }{ + {name: "dependent", want: store.ErrCloneSnapshotInUse}, + {name: "list-failed", listErr: errAPIServerUnavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := t.Context() + backend := store.NewInMemory() + seedOwnedCloneSnapshot(t, backend) + + if err := backend.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "third", + Props: map[string]string{store.RestoreFromSnapshotProp: "src:clone-dst"}, + }); err != nil { + t.Fatalf("seed the restored definition: %v", err) + } + + var seen bool + + err := store.ReapClonedSnapshot(ctx, + markSeenAtListStore{Store: backend, seen: &seen, listErr: tc.listErr}, + store.ClonedSnapshotRef{Source: "src", Snapshot: "clone-dst", Clone: "dst"}) + if err == nil || (tc.want != nil && !errors.Is(err, tc.want)) { + t.Fatalf("reap = %v, want a refusal", err) + } + + snap, err := backend.Snapshots().Get(ctx, "src", "clone-dst") + if err != nil { + t.Fatalf("the kept snapshot is gone: %v", err) + } + + if mark, ok := snap.Props[store.CloneSnapshotReapingProp]; ok { + t.Errorf("the kept snapshot still carries the reaping mark %q", mark) + } + }) + } +} diff --git a/pkg/store/inmemory_resource_definition.go b/pkg/store/inmemory_resource_definition.go index 06234153..6270d03d 100644 --- a/pkg/store/inmemory_resource_definition.go +++ b/pkg/store/inmemory_resource_definition.go @@ -33,6 +33,11 @@ type inMemoryResourceDefinitions struct { m map[string]apiv1.ResourceDefinition } +// ListUncached has nothing to bypass here: this store is the API server. +func (s *inMemoryResourceDefinitions) ListUncached(ctx context.Context) ([]apiv1.ResourceDefinition, error) { + return s.List(ctx) +} + func (s *inMemoryResourceDefinitions) List(_ context.Context) ([]apiv1.ResourceDefinition, error) { s.mu.RLock() defer s.mu.RUnlock() diff --git a/pkg/store/k8s/deleting_flag_internal_test.go b/pkg/store/k8s/deleting_flag_internal_test.go index faf3d437..a5cf6972 100644 --- a/pkg/store/k8s/deleting_flag_internal_test.go +++ b/pkg/store/k8s/deleting_flag_internal_test.go @@ -169,3 +169,31 @@ func TestCrdToWireResourceSurfacesDeletingFlag(t *testing.T) { t.Fatalf("crdToWireResource surfaced DELETE on a live Resource") } } + +// A snapshot delete only stamps a DeletionTimestamp while a satellite +// finalizer holds it, and a restore that read it as live was restored from +// data being destroyed. +func TestCrdToWireSnapshotSurfacesDeletingFlag(t *testing.T) { + t.Parallel() + + now := metav1.NewTime(time.Now()) + crd := &crdv1alpha1.Snapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "src.clone-dst", + DeletionTimestamp: &now, + Finalizers: []string{"blockstor.cozystack.io/satellite-snapshot"}, + }, + Spec: crdv1alpha1.SnapshotSpec{ResourceDefinitionName: "src", SnapshotName: "clone-dst"}, + } + + if wire := crdToWireSnapshot(crd, nil); !slices.Contains(wire.Flags, apiv1.SnapshotFlagDelete) { + t.Fatalf("crdToWireSnapshot did not surface DELETE on a deleting Snapshot: flags=%v", wire.Flags) + } + + live := &crdv1alpha1.Snapshot{ + Spec: crdv1alpha1.SnapshotSpec{ResourceDefinitionName: "src", SnapshotName: "snap"}, + } + if slices.Contains(crdToWireSnapshot(live, nil).Flags, apiv1.SnapshotFlagDelete) { + t.Fatalf("crdToWireSnapshot surfaced DELETE on a live Snapshot") + } +} diff --git a/pkg/store/k8s/rd_list_uncached_test.go b/pkg/store/k8s/rd_list_uncached_test.go new file mode 100644 index 00000000..4815d0b7 --- /dev/null +++ b/pkg/store/k8s/rd_list_uncached_test.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 + +package k8s_test + +import ( + "context" + "testing" + + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + crdv1alpha1 "github.com/cozystack/blockstor/api/v1alpha1" + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store/k8s" +) + +// blindToDefinitions is a client whose cache has seen no definition at all. +type blindToDefinitions struct{ ctrlclient.Client } + +func (b blindToDefinitions) List(ctx context.Context, list ctrlclient.ObjectList, opts ...ctrlclient.ListOption) error { + if _, ok := list.(*crdv1alpha1.ResourceDefinitionList); ok { + return nil + } + + return b.Client.List(ctx, list, opts...) //nolint:wrapcheck // pass-through +} + +// ListUncached exists for a decision that destroys something when the answer +// comes back short, so it has to reach past a cache that trails. +func TestResourceDefinitionListUncachedReadsPastTheCache(t *testing.T) { + if fixture == nil { + t.Skip("envtest assets not installed; run `make setup-envtest` to enable") + } + + ctx := t.Context() + + if err := k8s.New(fixture.client).ResourceDefinitions().Create(ctx, + &apiv1.ResourceDefinition{Name: "pvc-uncached-list"}); err != nil { + t.Fatalf("seed definition: %v", err) + } + + t.Cleanup(func() { + _ = k8s.New(fixture.client).ResourceDefinitions().Delete(context.Background(), "pvc-uncached-list") + }) + + st := k8s.NewWithAPIReader(blindToDefinitions{fixture.client}, fixture.client) + + cached, err := st.ResourceDefinitions().List(ctx) + if err != nil || len(cached) != 0 { + t.Fatalf("fixture: the cached list saw %d definition(s) (%v), want none", len(cached), err) + } + + live, err := st.ResourceDefinitions().ListUncached(ctx) + if err != nil { + t.Fatalf("ListUncached: %v", err) + } + + found := false + + for i := range live { + if live[i].Name == "pvc-uncached-list" { + found = true + } + } + + if !found { + t.Errorf("ListUncached missed a definition the API server holds: %d returned", len(live)) + } +} diff --git a/pkg/store/k8s/resource_definitions.go b/pkg/store/k8s/resource_definitions.go index 1b8a9984..2a0e742c 100644 --- a/pkg/store/k8s/resource_definitions.go +++ b/pkg/store/k8s/resource_definitions.go @@ -51,21 +51,17 @@ type resourceDefinitions struct { } func (s *resourceDefinitions) List(ctx context.Context) ([]apiv1.ResourceDefinition, error) { - var crdList crdv1alpha1.ResourceDefinitionList - - err := s.c.List(ctx, &crdList) - if err != nil { - return nil, errors.Wrap(err, "list ResourceDefinition CRDs") - } + return s.listThrough(ctx, s.c) +} - out := make([]apiv1.ResourceDefinition, 0, len(crdList.Items)) - for i := range crdList.Items { - out = append(out, crdToWireRD(&crdList.Items[i])) +// ListUncached lists through the direct reader when one is wired, and through +// the cache otherwise, where the cache is all there is. +func (s *resourceDefinitions) ListUncached(ctx context.Context) ([]apiv1.ResourceDefinition, error) { + if s.apiReader == nil { + return s.listThrough(ctx, s.c) } - sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) - - return out, nil + return s.listThrough(ctx, s.apiReader) } func (s *resourceDefinitions) Get(ctx context.Context, name string) (apiv1.ResourceDefinition, error) { @@ -307,6 +303,24 @@ func (s *resourceDefinitions) Delete(ctx context.Context, name string) error { return nil } +func (s *resourceDefinitions) listThrough(ctx context.Context, reader ctrlclient.Reader) ([]apiv1.ResourceDefinition, error) { + var crdList crdv1alpha1.ResourceDefinitionList + + err := reader.List(ctx, &crdList) + if err != nil { + return nil, errors.Wrap(err, "list ResourceDefinition CRDs") + } + + out := make([]apiv1.ResourceDefinition, 0, len(crdList.Items)) + for i := range crdList.Items { + out = append(out, crdToWireRD(&crdList.Items[i])) + } + + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + + return out, nil +} + // getForPatch reads the object a patch starts from. A cache NotFound is // re-read live, the way Get does: a caller that patches a definition it has // just created reads it at the peak of the informer's lag, and taking the diff --git a/pkg/store/k8s/snapshots.go b/pkg/store/k8s/snapshots.go index 5a040ba3..40573f21 100644 --- a/pkg/store/k8s/snapshots.go +++ b/pkg/store/k8s/snapshots.go @@ -351,6 +351,14 @@ func crdToWireSnapshot( copy(out.Flags, crd.Status.Flags) } + // A delete only stamps a DeletionTimestamp while a satellite finalizer + // holds the snapshot, and that lasts until the on-disk delete succeeds. + // Served as a plain snapshot, a restore read it as live and was restored + // from data being destroyed. Same flag resources carry for the same state, + // and like theirs it outranks every other State: the REST view stamps no + // SUCCESSFUL beside it. + out.Flags = withDeletingFlag(out.Flags, crd.DeletionTimestamp != nil) + if len(crd.Spec.VolumeDefinitions) > 0 { out.VolumeDefinitions = wireVolumeDefinitions(crd.Spec.VolumeDefinitions, vdPropsByNumber) } diff --git a/pkg/store/local_props.go b/pkg/store/local_props.go index 31065b60..948edd29 100644 --- a/pkg/store/local_props.go +++ b/pkg/store/local_props.go @@ -61,15 +61,15 @@ const RestoreAdoptedProp = "Blockstor/RestoreAdopted" // the snapshot it was restored from held, as `=` pairs. // // The snapshot is the reference a retry judges a leftover against, and it does -// not always outlive the leftover: an operator can delete it. Judged without -// it, a definition with one volume and one live replica read as finished -// whatever the snapshot held, so a clone that had restored only some of its -// volumes before the snapshot went was reported complete. The record is -// written with the marker, and it is the reference whenever the caller does -// not hand the snapshot in, whether or not the snapshot still exists: every -// rollback's spare-or-delete judgement (ReadsAsFinished) reads it. A leftover -// written before the record existed carries none and is judged as before, on -// its own volumes alone. +// not always outlive the leftover: an operator can delete it, and a clone's +// internal one is reaped. Judged without it, a definition with one volume and +// one live replica read as finished whatever the snapshot held, so a clone +// that had restored only some of its volumes before the snapshot went was +// reported complete. The record is written with the marker, and it is the +// reference whenever the caller does not hand the snapshot in, whether or not +// the snapshot still exists: every rollback's spare-or-delete judgement +// (ReadsAsFinished) reads it. A leftover written before the record existed +// carries none and is judged as before, on its own volumes alone. const RestoreVolumesProp = "Blockstor/RestoreVolumes" // WithRestoreMarker stamps a definition being restored from snap with the diff --git a/pkg/store/store.go b/pkg/store/store.go index f43d5732..173772c6 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -177,6 +177,13 @@ type ResourceGroupStore interface { // ResourceGroupStore (nil = untouched, empty = clear). type ResourceDefinitionStore interface { List(ctx context.Context) ([]apiv1.ResourceDefinition, error) + + // ListUncached answers the same question from the API server when the + // store has a direct reader, bypassing a cache that can trail a create. + // For a decision that destroys something when the answer comes back + // short, like reaping a snapshot no definition seems to use. + ListUncached(ctx context.Context) ([]apiv1.ResourceDefinition, error) + Get(ctx context.Context, name string) (apiv1.ResourceDefinition, error) Create(ctx context.Context, rd *apiv1.ResourceDefinition) error Update(ctx context.Context, rd *apiv1.ResourceDefinition) error