diff --git a/.github/workflows/contract.yml b/.github/workflows/contract.yml index 670f4c60..ae29fd63 100644 --- a/.github/workflows/contract.yml +++ b/.github/workflows/contract.yml @@ -17,6 +17,10 @@ on: # merges still exercise contract tests. push: branches: [main] + # The site under site/ is published on its own and has no tests. + paths-ignore: + - 'site/**' + - 'netlify.toml' permissions: {} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 197442fd..3d68099e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -16,6 +16,10 @@ on: # merges still exercise the envtest harness. push: branches: [main] + # The site under site/ is published on its own and has no tests. + paths-ignore: + - 'site/**' + - 'netlify.toml' permissions: {} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 69a8a32f..3a8fd1cc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,6 +7,10 @@ name: Lint on: push: branches: [main] + # The site under site/ is published on its own and has no tests. + paths-ignore: + - 'site/**' + - 'netlify.toml' permissions: {} diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 00000000..4d54f85e --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,84 @@ +name: Deploy site to Pages + +# blockstor.org is the Hugo and Docsy site under site/. Netlify builds the +# deploy previews from the same sources (netlify.toml). +on: + push: + branches: [main] + paths: + - 'site/**' + - '.github/workflows/pages.yml' + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +# One deployment at a time. A queued run is skipped for the newest one, but +# a deployment in progress is allowed to finish. +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + env: + HUGO_VERSION: 0.164.0 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install Hugo + run: | + wget -q -O "${RUNNER_TEMP}/hugo.deb" \ + "https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb" + sudo dpkg -i "${RUNNER_TEMP}/hugo.deb" + + - name: Setup Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version: "1.23" + cache: false + + - name: Setup Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + # Hugo runs Node asset tools with "node --permission", which needs + # Node 22.13 or later. + node-version: "22" + + - name: Setup Pages + id: pages + uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 + + - name: Install Node.js dependencies + working-directory: site + run: npm ci + + - name: Build with Hugo + working-directory: site + env: + HUGO_ENVIRONMENT: production + HUGO_ENV: production + BASE_URL: ${{ steps.pages.outputs.base_url }} + run: hugo --gc --minify --baseURL "${BASE_URL}/" + + - name: Upload artifact + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + with: + path: ./site/public + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index d97aedb8..8417d2ac 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -26,6 +26,10 @@ name: Pull Request on: pull_request: types: [opened, synchronize, reopened] + # The site under site/ is published on its own and has no tests. + paths-ignore: + - 'site/**' + - 'netlify.toml' # Cancel in-flight runs for the same PR when a new push arrives — saves # runner minutes on rapid force-push iterations. diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index ed62a118..82493ae4 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -7,6 +7,10 @@ name: E2E Tests on: push: branches: [main] + # The site under site/ is published on its own and has no tests. + paths-ignore: + - 'site/**' + - 'netlify.toml' permissions: {} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ab968f7a..5084e743 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,10 @@ name: Tests on: push: branches: [main] + # The site under site/ is published on its own and has no tests. + paths-ignore: + - 'site/**' + - 'netlify.toml' permissions: {} diff --git a/.gitignore b/.gitignore index bebae282..32e8dace 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,9 @@ .claude/ pkg/satellite/pvc-dispatch-adjust.res linstor-migrate + +# blockstor.org site build output +site/public/ +site/resources/ +site/node_modules/ +site/.hugo_build.lock diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b616a11..bb337e81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,7 +80,7 @@ Release-gate hardening release. A full independent acceptance gate (default NO-G ### Known issues -- **Concurrent rapid late `vd c` (BUG-048; "Bug 50" in `docs/known-issues.md`)** — two back-to-back manual `volume-definition create` calls on an existing multi-replica RD can intermittently drop or wedge the second volume. Operator-only (NOT reachable via CSI, which creates exactly one volume-definition per resource-definition), availability-only (no data loss, no node-reboot deadlock), recoverable by deleting and re-adding the volume-definition sequentially. A code fix is in progress. +- **Concurrent rapid late `vd c` (BUG-048; "Bug 50" in `docs/known-issues.md`, since removed)** — two back-to-back manual `volume-definition create` calls on an existing multi-replica RD can intermittently drop or wedge the second volume. Operator-only (NOT reachable via CSI, which creates exactly one volume-definition per resource-definition), availability-only (no data loss, no node-reboot deadlock), recoverable by deleting and re-adding the volume-definition sequentially. A code fix is in progress. ### Testing & infrastructure diff --git a/docs/bug-42-investigation.md b/docs/bug-42-investigation.md index ac3281d0..e257f818 100644 --- a/docs/bug-42-investigation.md +++ b/docs/bug-42-investigation.md @@ -2,7 +2,7 @@ ## TL;DR -The premise in `known-issues.md` ("satellite pod IPs come from outside the pod CIDR" / "netInterfaces[0].address=") is wrong. On both e2e-iptables and e2e-quorum the stored `Node.spec.netInterfaces[0].address` is the host InternalIP (10.27.0.x / 10.161.0.x). What actually differs between the two stands is **who owns the Node CRD's `spec` once piraeus-operator finishes reconciling**: +The premise in the Bug 42 entry of `known-issues.md` (since removed) ("satellite pod IPs come from outside the pod CIDR" / "netInterfaces[0].address=") is wrong. On both e2e-iptables and e2e-quorum the stored `Node.spec.netInterfaces[0].address` is the host InternalIP (10.27.0.x / 10.161.0.x). What actually differs between the two stands is **who owns the Node CRD's `spec` once piraeus-operator finishes reconciling**: - **e2e-quorum** — piraeus runs against its own bundled `linstor-controller` (LinstorCluster `spec: {}`). Piraeus never touches the blockstor Node CRD. Spec stays exactly as `install-blockstor.sh` left it: one interface `default` with the k8s InternalIP, `generation: 1`, no `Aux/piraeus.io/*` props. - **e2e-iptables** — `LinstorCluster.spec.externalController.url = http://blockstor-apiserver.blockstor-system.svc:3370` was set (by an `observability-*` e2e scenario, see `tests/e2e/observability-three-way.sh:78-84` and `tests/e2e/observability-capacity-correlation.sh:240-244`). The operator now drives blockstor-apiserver as its LINSTOR backend and rewrites the Node CRD via `PUT /v1/nodes/{node}/net-interfaces/default-ipv4`: interface name changes from `default` to `default-ipv4`, `satellitePort=3366` + `satelliteEncryptionType=PLAIN` are added, and `Aux/piraeus.io/last-applied` / `Aux/piraeus.io/configured-interfaces` props appear in `spec.props`. Generation climbs to 9. @@ -116,5 +116,5 @@ Both `tests/e2e/observability-three-way.sh:81-84` and `tests/e2e/observability-c ## Recommended next steps 1. Apply fix (A) + (B), each as its own commit. -2. Rewrite `docs/known-issues.md` Bug 42 entry to match what's actually happening (interface-name + props churn from externalController-mode piraeus), drop the unsupported pod-CIDR claim, and adjust severity (P3 — cosmetic / fragility, not a DRBD outage). +2. Rewrite the Bug 42 entry (the page has since been removed) to match what's actually happening (interface-name + props churn from externalController-mode piraeus), drop the unsupported pod-CIDR claim, and adjust severity (P3 — cosmetic / fragility, not a DRBD outage). 3. Optional follow-up: add a contract test asserting that `snapshot_fetcher.peerAddr()` resolves correctly when the only NetInterface is named `default-ipv4`. That pins fix (A) and would have caught this earlier. diff --git a/docs/cli-parity-known-deltas.md b/docs/cli-parity-known-deltas.md index 698670d0..d84fdeef 100644 --- a/docs/cli-parity-known-deltas.md +++ b/docs/cli-parity-known-deltas.md @@ -60,6 +60,7 @@ Row IDs match the command-catalogue indexes used by `cli-parity-refresh.sh` (see | 42 | `r d ` (idempotent resource delete) | WIRE_SHAPE | permanent | BLOCKSTOR idempotent-delete envelope. Deleting a resource placement on a node that holds none returns `SUCCESS: resource already absent: on `; upstream LINSTOR returns `WARNING: Node: , Resource: not found`. Both exit 0. Deliberate: a CSI `DeleteVolume` / operator retry on an already-removed placement MUST be idempotent. Same family as the snapshot-delete idempotency in row 33. Pinned by the resource-delete REST handler's already-absent path (`pkg/rest/resources.go`). | | 40 | `n c --node-type Satellite` (node-create success envelope) | WIRE_SHAPE | permanent | BLOCKSTOR envelope shape on satellite node registration. BS emits `node created: ` + a `SUCCESS: No active connection to satellite ''` line; upstream emits `New node '' registered.` (with a UUID detail) + a `WARNING: No active connection to satellite ''` line whose Details explain the controller will (re-)establish the connection. Both register the node and exit 0; the operator-visible outcome (node exists, awaiting satellite handshake) is identical. The "no active connection" notice is INFO/SUCCESS-class in BS vs WARNING-class upstream, and BS does not surface the volatile node UUID inline. Envelope-shape only; no behavioural divergence. | | 16 | `ps l` (physical-storage list — `size` field) | WIRE_SHAPE | 2026-12-31 | BS `/v1/physical-storage` omits the per-device `size` key that python-linstor's `show_physical_storage` reads unconditionally (`linstor/responses.py` `devices.size`), so `linstor ps l` raises `KeyError: 'size'` and exits 2 against BS where upstream renders the table and exits 0. `ps l` is a hardware-discovery convenience surface; linstor-csi / piraeus-operator never call it, and the device-pool creation path (`ps create-device-pool`, cli-matrix `ps-cdp-*`) is unaffected. Tracked as a missing wire field to populate on the physical-storage DTO; whitelisted until then. | +| 86 | `node interface modify --active` | BEHAVIOR | permanent | blockstor has no satellite-to-controller connection to re-target: the satellite talks only to the Kubernetes API through the in-cluster Service. `NodeNetInterface` carries no `Active` field, and the REST `is_active` flag is synthesised on read as "first interface", so `--active` changes nothing. To make another interface active, reorder `spec.netInterfaces` so it comes first; a changed address reaches the node's DRBD configuration after its satellite pod restarts. Pinned by `TestSatelliteFlagsLackControllerBindAddress` and `TestNodeCRDHasNoActiveField` in `pkg/satellite/stream/redial_spec_test.go`. | ## Open (block merge until addressed) diff --git a/docs/csi-sanity-classification.md b/docs/csi-sanity-classification.md index c7150a1e..2c884779 100644 --- a/docs/csi-sanity-classification.md +++ b/docs/csi-sanity-classification.md @@ -10,7 +10,7 @@ Fix (this PR): relax the apiserver validator to mirror upstream, AND make the Re The residual failures are unchanged from the table below (all class (a) Node/satellite-convergence + class (b) apiserver scale-out cache race + Bug 33 satellite-snapshot finalizer drain in AfterEach teardown — the latter affects lowercase snapshots too). -Prior baseline (pre-audit): **56/74 specs pass** (74 ran; 17 skipped; 1 pending) after seven wire-shape gaps and three concurrency races were closed. The same job-config is used here. The audit collected 9 hard failures during the first 16 seconds of the run before the harness wedged on a snapshot-data-plane test (`sanity-controller-source-vol-*`) waiting for a satellite-side reconcile that never converged on the e2e-lc1 stand (Bug 33 territory — see `docs/known-issues.md`). Wire-shape and routing observations cover all 18 expected failures from the prior baseline. +Prior baseline (pre-audit): **56/74 specs pass** (74 ran; 17 skipped; 1 pending) after seven wire-shape gaps and three concurrency races were closed. The same job-config is used here. The audit collected 9 hard failures during the first 16 seconds of the run before the harness wedged on a snapshot-data-plane test (`sanity-controller-source-vol-*`) waiting for a satellite-side reconcile that never converged on the e2e-lc1 stand (Bug 33 territory, since closed by the orphan DRBD sweeper). Wire-shape and routing observations cover all 18 expected failures from the prior baseline. ## Failure-class legend @@ -60,6 +60,6 @@ So the **post-fix expected pass rate stays at 56/74**. The new finding from this 2. **CreateSnapshot empty-name validation** (c, #12-#13) — single check, flips two FAILs to PASS. Surfaces a real bug-class (silent slug-of-empty). 3. **DeleteSnapshot folds unknown-RD into success** (b, #14) — one-liner, flips one FAIL to PASS, closes a CSI idempotence corner. 4. **ListSnapshots empty-result envelope + exact-fit pagination** (c, #15-#16) — small handler tweaks, flips two FAILs to PASS. -5. **Bug 33 satellite-finalizer storm** — not a csi-sanity gap per se, but the audit blocker for re-running the suite cleanly. Already tracked in `docs/known-issues.md` (Bug 33 — closed by 5.34 sweeper; the storm seen on this run was a residual from the controller crashloop before Bug 41 RBAC was applied to the live stand). +5. **Bug 33 satellite-finalizer storm** — not a csi-sanity gap per se, but the audit blocker for re-running the suite cleanly. Tracked as Bug 33 (closed by 5.34 sweeper; the storm seen on this run was a residual from the controller crashloop before Bug 41 RBAC was applied to the live stand). After (1)–(4) the expected pass rate is **64/74**. The remaining 10 failures are all (a) (kernel-mount Node Service + satellite-convergence snapshot/clone tests) and are out of scope for the single-pod csi-sanity harness; running them needs a real piraeus-operator deployment with a CSI Node sidecar per worker and a stand with reliable satellite reconcile. diff --git a/docs/e2e-audit.md b/docs/e2e-audit.md index 5cce8a35..a032c9b2 100644 --- a/docs/e2e-audit.md +++ b/docs/e2e-audit.md @@ -38,7 +38,7 @@ Each row names the Tier 2 test(s) that exercise the same REST/CRD/reconciler pat | `placement-label-sync.sh` | NodeLabelSyncReconciler mirroring k8s labels → `Aux/...` props; envtest IS real k8s API. | Group A `TestGroupANodeAuxLabelSync` | | `recovery-false-diskless.sh` | Today asserts 404 on `r mkavail` (handler not wired). Pure wire-shape probe — no kernel-state contract. | Group F wire-shape probes; will move to Tier 4 when the handler lands and the test becomes "adopt existing on-disk state" | -`lc-rd-delete-churn.sh` is **kept** despite touching the same handlers — it runs 10 iterations and watches `dmesg`/`zfs list` for orphan ZVOLs and `node_id of X instead of Y` kernel warnings (cited by `docs/known-issues.md` as the validation source for the storage-sweeper). That assertion needs a live cluster. +`lc-rd-delete-churn.sh` is **kept** despite touching the same handlers — it runs 10 iterations and watches `dmesg`/`zfs list` for orphan ZVOLs and `node_id of X instead of Y` kernel warnings (the validation source for the storage-sweeper). That assertion needs a live cluster. --- diff --git a/docs/known-issues.md b/docs/known-issues.md deleted file mode 100644 index 899333d0..00000000 --- a/docs/known-issues.md +++ /dev/null @@ -1,323 +0,0 @@ -# Known Issues - -This document tracks outstanding bugs and observations surfaced during the scenario/e2e validation session. Each entry references the scenario or test where it was first observed, plus the closing commit if any. - -Severity legend: - -- **P0** — data corruption, split-brain, lost volumes, or cluster-wide unavailability -- **P1** — feature unusable, retry/cancel paths missing, regression vs upstream LINSTOR -- **P2** — cosmetic, log noise, doc/RBAC drift, ergonomic gaps - -## Bug 32: force-strip ZVOL cleanup race - -**Status**: open (observation, not a defect per se) -**Severity**: P2 -**Scenario reference**: tests/scenarios/05-finalizers.md §5.34 -**Surfaced by**: scenario -**Reproduction steps**: - -1. Create a Resource with a ZFS-backed VolumeDefinition on a single node. -2. Force-delete the Resource CR (`kubectl delete --grace-period=0 --force`) while the satellite is still mid-`drbdsetup down`. -3. Observe the underlying ZVOL on the host (`zfs list -t volume`). - -**Expected behaviour**: ZVOL is destroyed in lock-step with the DRBD resource teardown; no stray dataset survives the CR removal. - -**Actual behaviour**: A short window exists where the CR is gone but the ZVOL still resides on disk because the satellite finalizer was bypassed. The next reconcile or operator restart eventually GCs it, but for a few seconds the storage pool reports phantom capacity. - -**Recommended fix**: No code change required. Document the window and rely on Bug 33 (DRBD orphan sweeper) plus Bug 43 (storage orphan sweeper) to handle the residual cleanup deterministically. - -**Related commits / tests**: see Bug 33 (DRBD sweeper) and Bug 43 (storage sweeper). - -## Bug 33: orphan kernel DRBD-Diskless after force-strip - -**Status**: closed (by Bug 5.34 sweeper) -**Severity**: P1 -**Scenario reference**: tests/scenarios/05-finalizers.md §5.34 -**Surfaced by**: scenario -**Reproduction steps**: - -1. Create a 3-replica Resource (2 disk + 1 diskless tiebreaker). -2. Force-delete the tiebreaker Resource CR before the satellite finalizer drains. -3. Inspect `drbdsetup status` on the former diskless node. - -**Expected behaviour**: Kernel DRBD device on the diskless node is fully removed; `drbdsetup status` no longer lists it. - -**Actual behaviour**: Kernel resource remained in `Diskless` state with no corresponding CR, preventing subsequent recreation on the same node (port/minor collision). - -**Recommended fix**: Periodic sweeper on the satellite enumerates kernel DRBD resources and reconciles against the local CR cache; any kernel resource without a matching CR is `drbdsetup down`-ed. - -**Related commits / tests**: 5.34 sweeper landed in the satellite controller; e2e covers via scenario 05 force-strip path. - -## Bug 34: migrate-disk endpoint missing - -**Status**: closed (by 05b8709); Option B follow-up open -**Severity**: P1 -**Scenario reference**: tests/scenarios/07-toggle-disk.md §7.4 -**Surfaced by**: e2e test -**Reproduction steps**: - -1. Create a 2-replica Resource on nodes A and B. -2. Issue `linstor resource-definition migrate-disk `. - -**Expected behaviour**: REST handler accepts the request, schedules a synchronous disk migration (toggle-disk on, then off on source) without losing redundancy. - -**Actual behaviour** (pre-fix): Endpoint returned 404; the upstream LINSTOR client could not orchestrate disk migration through the apiserver. - -**Recommended fix (landed)**: REST handler `migrate-disk` wired in 05b8709 to a two-step toggle-disk pipeline (add on target → wait UpToDate → remove on source). - -**Option B follow-up (open)**: Wrap the two-step sequence in a single reconciler-driven state machine so an apiserver restart mid-migration does not strand the user with an extra diskful replica. Track at next sprint. - -**Related commits / tests**: 05b8709; scenario 07 §7.4 covers the happy path. - -## Bug 35: placer FreeCapacity gate missing - -**Status**: open -**Severity**: P0 -**Scenario reference**: tests/scenarios/08-placer.md §8.2 -**Surfaced by**: scenario -**Reproduction steps**: - -1. Create two storage pools with vastly different free capacity (e.g. 100 GiB and 1 GiB free). -2. Create a Resource of size 10 GiB without an explicit node selector. - -**Expected behaviour**: Placer rejects the small pool because the requested size exceeds its FreeCapacity and chooses the larger pool (or fails fast with a clear error). - -**Actual behaviour**: Placer ignores FreeCapacity entirely and may select the smaller pool, leading to `ENOSPC` at ZVOL/LV create time and a stuck Resource in `Creating`. - -**Recommended fix**: Add a hard FreeCapacity gate inside the candidate-pool filter before scoring. Treat pools with `free < requested + reservedHeadroom` as ineligible. Add a placer unit test with two pools at different free sizes. - -**Related commits / tests**: none yet; blocker for placer 1.0. - -## Bug 36: handleVDUpdate wholesale-replace - -**Status**: open -**Severity**: P1 -**Scenario reference**: tests/scenarios/09-vd-update.md §9.1 -**Surfaced by**: agent -**Reproduction steps**: - -1. Create a VolumeDefinition with `props: {a: 1, b: 2}`. -2. PATCH the VD with `override_props: {a: 9}` (intent: change only `a`). - -**Expected behaviour**: Final props are `{a: 9, b: 2}` — only the keys named in `override_props` are touched. - -**Actual behaviour**: `handleVDUpdate` rewrites the entire props map from the request body, so `b` is silently dropped. - -**Recommended fix**: Replace the wholesale assignment with a merge: apply `override_props` keys on top of the existing map, then remove keys named in `delete_props`. Mirrors the upstream LINSTOR controller semantics and is the precondition for Bug 37. - -**Related commits / tests**: none; depends on Bug 37. - -## Bug 37: VolumeDefinitionModify override_props/delete_props no-op - -**Status**: open -**Severity**: P1 -**Scenario reference**: tests/scenarios/09-vd-update.md §9.2 -**Surfaced by**: e2e test -**Reproduction steps**: - -1. Apply `VolumeDefinitionModify` with `override_props: {DrbdOptions/Net/protocol: C}`. -2. GET the VD and inspect `props`. - -**Expected behaviour**: `DrbdOptions/Net/protocol` is set to `C`; `delete_props` removes named keys. - -**Actual behaviour**: Both fields are accepted by the REST schema but ignored by the handler — the props map is unchanged. - -**Recommended fix**: Wire `override_props` and `delete_props` through the same merge path introduced by Bug 36's fix. Add e2e assertions that round-trip both add and remove. - -**Related commits / tests**: depends on Bug 36; scenario 09 §9.2 has the assertion stub. - -## Bug 38: VD shrink accepted without STATE_INFO warning - -**Status**: open -**Severity**: P2 (cosmetic) -**Scenario reference**: tests/scenarios/09-vd-update.md §9.3 -**Surfaced by**: scenario -**Reproduction steps**: - -1. Create a VD of size 10 GiB. -2. PATCH the VD to size 5 GiB. - -**Expected behaviour**: Either the request is rejected, or it is accepted with a `STATE_INFO` / warning in the response indicating that shrinking is unsafe and won't reclaim space on the backing pool until the FS is also shrunk. - -**Actual behaviour**: Request is accepted silently; no warning surfaces to the operator. Backing volume keeps its original allocation. - -**Recommended fix**: Append a `STATE_INFO` entry to the API response when `newSize < currentSize`. No semantic change to storage; purely advisory. - -**Related commits / tests**: none. - -## Bug 39: toggle-disk retry counter missing - -**Status**: open -**Severity**: P1 -**Scenario reference**: tests/scenarios/07-toggle-disk.md §7.6 -**Surfaced by**: agent -**Reproduction steps**: - -1. Initiate `toggle-disk` on a Resource where the target node's storage pool is briefly unavailable. -2. Observe reconciler logs. - -**Expected behaviour**: Reconciler retries with bounded backoff and a visible retry counter on the Resource status; after N attempts it surfaces a terminal error condition. - -**Actual behaviour**: Reconciler retries forever with no visibility; if the failure is permanent, the Resource is stuck "in flight" indefinitely. - -**Recommended fix**: Add `status.toggleDisk.attempts` and `status.toggleDisk.lastError`. Cap at e.g. 10 attempts before transitioning to `Failed` with a clear condition. Couple with Bug 40. - -**Related commits / tests**: none; precondition for the migrate-disk Option B state machine. - -## Bug 40: toggle-disk cancel state machine missing - -**Status**: open -**Severity**: P1 -**Scenario reference**: tests/scenarios/07-toggle-disk.md §7.7 -**Surfaced by**: agent -**Reproduction steps**: - -1. Start a long-running `toggle-disk` (e.g. large initial sync). -2. Issue a cancel/abort via the REST API. - -**Expected behaviour**: Reconciler observes the cancel intent, rolls back the partial change (removes the half-added diskful, or re-adds the half-removed one) and returns the Resource to its pre-toggle state. - -**Actual behaviour**: No cancel endpoint exists; the only escape is to delete the Resource entirely. - -**Recommended fix**: Model toggle-disk as an explicit state machine (`Pending → Syncing → Promoting → Done | Cancelling → Rolled-back`). Add a cancel verb in the REST surface and have the reconciler honour it idempotently. - -**Related commits / tests**: none; pairs with Bug 39. - -## Bug 41: RBAC nodes get/list/watch missing in stand yaml - -**Status**: closed (13722e1) -**Severity**: P2 -**Scenario reference**: tests/scenarios/01-bootstrap.md §1.3 -**Surfaced by**: e2e test (stand bring-up) -**Reproduction steps**: - -1. Deploy blockstor on the dev stand with the previous controller-manifests yaml. -2. Watch controller logs at startup. - -**Expected behaviour**: Controller starts and lists Node objects to seed its topology cache. - -**Actual behaviour** (pre-fix): `nodes "" is forbidden: User "system:serviceaccount:..." cannot list resource "nodes" in API group "" at the cluster scope`; controller crashloops. - -**Recommended fix (landed)**: Add `nodes` verbs `get/list/watch` to the ClusterRole shipped with the stand yaml. - -**Related commits / tests**: 13722e1. - -## Bug 42: piraeus pod-CIDR drift on e2e-iptables - -**Status**: open (investigation pending) -**Severity**: P1 -**Scenario reference**: tests/scenarios/10-e2e-iptables.md §10.1 -**Surfaced by**: e2e test -**Reproduction steps**: - -1. Bring up the e2e-iptables Talos+QEMU stand via `make iter`. -2. Deploy piraeus-operator with the default values. -3. Wait for the satellite DaemonSet pods to schedule and the LINSTOR controller to register them. - -**Expected behaviour**: Satellite pod IPs come from the cluster's pod CIDR and are routable to/from the LINSTOR controller; resources reach `UpToDate` without DRBD net errors. - -**Actual behaviour**: Some satellite pods receive addresses outside the expected pod CIDR (suspected CNI bridge / kube-proxy iptables drift on the iptables-mode stand), DRBD peer connections flap, resources oscillate between `Connecting` and `Connected`. - -**Recommended fix**: Investigation pending. First steps: - -- Confirm whether the drift is at the CNI layer (bridge IPAM) or kube-proxy (SNAT/MASQUERADE on the iptables stand). -- Capture `ip a`, `iptables-save`, and `kubectl get pods -o wide` on a failing run. -- Compare against the ipvs-mode stand where the issue does not reproduce. - -**Related commits / tests**: scenario 10 §10.1 reproducer; no fix yet. - -## Bug 49: satellite ignores runtime NetInterface changes (deferred from scenario 3.10) - -**Status**: open (deferred — design gap, not a defect) -**Severity**: P2 -**Scenario reference**: tests/scenarios/03-networking.md §3.10 -**Surfaced by**: scenario audit (UG9 §"Managing network interface cards", lines 2167-2169) -**Reproduction steps**: - -1. `kubectl edit node.blockstor.io/` and re-order the entries under `spec.netInterfaces` (or change the first entry's `address`). -2. Observe the running satellite Pod on that worker — it does not re-dial anything, does not re-render its DRBD `on ` block, and continues talking to the kube-apiserver via the same Service VIP. -3. Restart the satellite Pod (`kubectl delete pod -l app=blockstor-satellite,kubernetes.io/hostname=`) — only on restart does the new `Spec.NetInterfaces[0].Address` flow into `LocalAddress` (via `$POD_IP`) and from there into the next reconcile-rendered .res file. - -**Expected behaviour (upstream LINSTOR contract)**: `linstor node interface modify ... --active` flips `StltConn/0/Active=true` and the satellite re-dials the controller via the freshly-active NIC without a process restart. - -**Actual behaviour (blockstor)**: No re-dial happens, and there is no field to flip. Two architectural reasons: - -1. Phase 10.6 retired the satellite→controller gRPC wire entirely. The satellite now talks to the kube-apiserver via the standard in-cluster config (`ctrl.GetConfig()` → `kubernetes.default.svc` Service VIP). There is no LINSTOR-style "controller endpoint" left to re-target. -2. `api/v1alpha1.NodeNetInterface` has no `Active` field. The `IsActive` flag on the REST wire is synthesised at conversion time (`i == 0` in `pkg/store/k8s/nodes.go`) — pure presentation, no behaviour. The only satellite code that reads peer NetInterfaces at runtime is `pkg/satellite/controllers/snapshot_fetcher.go` (peer-to-peer snapshot transport), and it picks by `Name == "default"` or first non-empty `Address`, not by an Active flag. - -**Recommended fix**: Deferred. Implementing Outcome A (live NIC switch) would require: - -- Adding an `Active` field to `NodeNetInterface` and a controller-runtime watch in the satellite that triggers reconciliation when `Spec.NetInterfaces[].Active` changes. -- A redirect plane that the satellite actually re-routes through — but the satellite's only outbound wire is the kube-apiserver REST/watch, which is owned by client-go and routes via the cluster Service VIP rather than this node's NIC selection. -- For the satellite→satellite snapshot stream the change would be marginal (peer-side already picks Address dynamically per Fetch call). - -Net: implementing the upstream contract requires either resurrecting a custom controller-bind wire (rejected by the Phase 10.6 design) or accepting that "active interface" is now a pure presentation field that operators must not rely on for connection re-targeting. - -The spec is pinned by `TestSatelliteFlagsLackControllerBindAddress` and `TestNodeCRDHasNoActiveField` in `pkg/satellite/stream/redial_spec_test.go`. If a future change implements Outcome A, both tests will fail and force the reviewer to replace them with a positive re-dial assertion. - -**Related commits / tests**: `pkg/satellite/stream/redial_spec_test.go`; tests/scenarios/03-networking.md §3.10. - -## Bug 43: mass-churn ZVOL leak on force-stripped finalizers - -**Status**: closed (by storage-orphan sweeper) -**Severity**: P0 -**Scenario reference**: tests/e2e/lc-rd-delete-churn.sh (scenario 5.4 extended — 10 RD rapid create/delete on one node) -**Surfaced by**: e2e test - -**Reproduction steps**: - -1. Run `lc-rd-delete-churn.sh` with `ITERS=10` against a 2-node stand. -2. Inject a satellite stall (e.g. SIGSTOP on the satellite pod) for the iteration's DRBD-down phase so the REST controller times out the per-iter delete and force-strips the Resource CRD finalizer. -3. After the loop, `zfs list -t volume` on each node. - -**Expected behaviour**: All 10 ZVOLs are destroyed in lock-step with the Resource CRD deletion; the final `zfs list -t volume` shows no `churn-N_NNNNN` datasets. - -**Actual behaviour** (pre-fix): 9 of 10 ZVOLs leak. The REST controller stamps `DeletionTimestamp` on the per-node Resource CRD; the satellite finalizer is mid-`drbdsetup down` so doesn't release the finalizer in time; the controller force-strips the finalizer; the satellite's `handleDelete` (which would have called `provider.DeleteVolume`) never runs; the ZVOL survives indefinitely. Bug 32 documents the short-window variant; Bug 43 is the deterministic mass-churn variant where the satellite genuinely cannot keep up and force-strip becomes the steady-state path. - -**Recommended fix (landed)**: New `StorageOrphanSweeperRunnable` in `pkg/satellite/controllers/storage_sweeper.go` periodically (every 5 minutes, same cadence as the DRBD orphan sweeper for Bug 33) walks every registered `storage.Provider` that implements the new `storage.VolumeLister` capability, diffs the on-disk volume list against the Resource CRDs scheduled to this node, and calls `provider.DeleteVolume` on each volume whose owning CRD has fully vanished (force-strip aftermath). - -Safety belts on the new path: - -- Wildcard-owned protection: a Resource CRD whose RD has been cascade-deleted out of order still counts as "owned" (volume number wildcard) so the sweeper never races `handleDelete`. -- Per-cycle delete bound (`StorageSweeperMaxDeletePerCycle = 3`) so a pathological upstream producer of orphans surfaces in logs without being silently cleaned in one tick. -- Storage-resource prefix allowlist (`pvc-`, `bs-`) so operator-created datasets that happen to share the pool are immune to the sweep. -- Per-Node opt-out annotation `blockstor.io/skip-orphan-storage-sweeper` for forensic-mode operators. -- `ListVolumeNames` skips ZFS deferred-delete markers (`__DELETED__`) and LVM snapshot LVs via the lv_attr column so the sweeper never confuses cleanup artefacts for orphans. - -Idempotency of `provider.DeleteVolume` was already guaranteed by Bug 33's contract; the sweeper relies on that — racing the satellite's own `handleDelete` is harmless because the second call returns nil. - -**Related commits / tests**: `pkg/satellite/controllers/storage_sweeper.go` + `pkg/satellite/controllers/storage_sweeper_test.go` (7 unit tests covering owned-leave-alone, orphan-reap, foreign-prefix immunity, per-node scope, rate-limit, skip-annotation, mid-delete-race protection, missing-RD wildcard protection); `pkg/storage/storage.go` adds the `VolumeLister` optional interface + `VolumeRef`; ZFS / LVM-thin / LVM-thick providers implement it. E2e validation via `lc-rd-delete-churn.sh` (passes 10 iters without ZVOL leak). - -## Bug 50: concurrent rapid late `vd c` wedges or drops the second volume - -**Status**: closed (#164 converge + #168 resize-deadlock guard; v0.1.14) (campaign tracking id: BUG-048) -**Severity**: P1 (availability; NOT data-loss, NOT a node-reboot deadlock; recoverable) -**Scenario reference**: tests/e2e/cli-matrix/multi-volume-late-vd-create.sh -**Surfaced by**: release-gate validation campaign -**Reproduction steps**: - -1. Create a multi-replica ResourceDefinition and let it converge (at least one VolumeDefinition, all replicas UpToDate). -2. Issue two `linstor volume-definition create ` calls back-to-back in rapid succession (concurrently, or scripted with no wait between them). - -**Expected behaviour**: Both volume-definitions are created; each new volume is laid down on every replica and reaches `UpToDate` (with one replica electing SyncSource), exactly as a sequential add would. - -**Actual behaviour**: Intermittently one of two faults occurs — (a) the second `vd c` returns SUCCESS but the VolumeDefinition is silently dropped (only one VD ends up persisted), or (b) the second volume is created but wedges `Inconsistent` with no SyncSource elected, so it never reaches `UpToDate`. In case (a) both calls report success. - -**Scope / reachability**: NOT reachable through the production CSI path. linstor-csi maps one PVC to one ResourceDefinition with exactly one VolumeDefinition, created in a single request; autoplace and `ControllerPublishVolume` add no VolumeDefinitions, and resize grows the existing volume in place. The fault is reachable only when an operator (or a script) issues concurrent / rapid back-to-back manual `vd c` calls against an existing multi-replica RD. No data is lost or corrupted (md5 held across all validation runs) and no node reboot is required. - -**Recovery**: Delete the wedged or missing volume-definition (`linstor vd d `) and re-add it sequentially — issue one `vd c`, wait for it to reach `UpToDate`, then issue the next. Always add late volume-definitions one at a time. - -**Recommended fix**: Serialise concurrent VolumeDefinition creation on a given ResourceDefinition (optimistic-concurrency precondition on the RD volume-minor allocation plus the auto-numbered VD create) so the second add observes the first, and seed the new volume's GI / current-UUID before peers attach so a deterministic SyncSource is elected. A first attempt (PR #164) reduced the silent-drop and fixed a day0 split-brain leg but regressed the CSI-reachable single-late-VD path, so it was not merged; a corrected fix that does not touch the single-VD path is tracked. - -**Related commits / tests**: cli-matrix cell `tests/e2e/cli-matrix/multi-volume-late-vd-create.sh` honestly catches this (as of #162); PR #164 (deferred). No closing commit yet. - -## Recommended next-fix order - -1. **Bug 35** (P0, placer FreeCapacity) — only P0; blocks placer 1.0 and risks ENOSPC at create time. -2. **Bug 42** (P1, piraeus pod-CIDR drift) — blocks the iptables-mode e2e lane. -3. **Bug 36 + 37** (P1, VD props merge) — fix together; 37 depends on 36's merge plumbing. -4. **Bug 39 + 40** (P1, toggle-disk retry/cancel) — fix together; together they unlock Bug 34's Option B state machine. -5. **Bug 50** (FIXED in v0.1.14, #164+#168) — serialise VolumeDefinition create on the RD and seed GI before peer attach; operator-only path, recoverable, not data-loss. Closed by #164 (converge) + #168 (resize-deadlock guard). -6. **Bug 34 Option B** (P1 follow-up) — wrap migrate-disk in the new state machine once 39/40 land. -7. **Bug 38** (P2, cosmetic STATE_INFO on shrink) — pure UX, do last. -8. **Bug 32** (P2, observation) — document only; no code fix needed. diff --git a/netlify.toml b/netlify.toml new file mode 100644 index 00000000..b5352b50 --- /dev/null +++ b/netlify.toml @@ -0,0 +1,37 @@ +# Netlify builds deploy previews of the blockstor.org site for pull +# requests. Production is GitHub Pages (.github/workflows/pages.yml). +# +# The site is Hugo with the Docsy theme under site/; base moves the build +# there, so every path below is relative to site/. + +[build] + base = "site" + publish = "public" + command = "npm install && hugo --gc --minify" + # Skip the preview when nothing under site/ or this file changed. The + # command runs from the base directory and exit 0 skips the build, so a + # missing or identical cached ref forces one. + ignore = ''' + if [ -z "$CACHED_COMMIT_REF" ] || [ "$CACHED_COMMIT_REF" = "$COMMIT_REF" ]; then exit 1; fi + git diff --quiet "$CACHED_COMMIT_REF" "$COMMIT_REF" -- . ../netlify.toml + ''' + +[build.environment] + HUGO_VERSION = "0.164.0" + HUGO_EXTENDED = "true" + NODE_VERSION = "22" + GO_VERSION = "1.23" + +[context.production.environment] + HUGO_ENV = "production" + +[context.deploy-preview] + command = "npm install && hugo --gc --minify -b $DEPLOY_PRIME_URL" + +[context.branch-deploy] + command = "npm install && hugo --gc --minify -b $DEPLOY_PRIME_URL" + +[[redirects]] + from = "/*" + to = "/404.html" + status = 404 diff --git a/pkg/satellite/stream/redial_spec_test.go b/pkg/satellite/stream/redial_spec_test.go index 5b9d7429..b29b2003 100644 --- a/pkg/satellite/stream/redial_spec_test.go +++ b/pkg/satellite/stream/redial_spec_test.go @@ -69,8 +69,8 @@ package stream_test // Tracking: // // - tests/scenarios/03-networking.md §3.10 -// - docs/known-issues.md → "Bug 49: satellite ignores runtime -// NetInterface changes (deferred from scenario 3.10)" +// - docs/cli-parity-known-deltas.md → row 86, "node interface +// modify --active" (formerly Bug 49, deferred from scenario 3.10) // // Outcome A (active re-dial loop) was considered and rejected: the // blockstor architecture does not have a satellite→controller dial diff --git a/site/README.md b/site/README.md new file mode 100644 index 00000000..0324fc1a --- /dev/null +++ b/site/README.md @@ -0,0 +1,11 @@ +# blockstor.org + +The project website and documentation, built with [Hugo](https://gohugo.io/) and the [Docsy](https://www.docsy.dev/) theme. Production is published to GitHub Pages from `main`; pull requests get a Netlify deploy preview. + +## Run it locally + +You need Hugo extended (0.161 or newer), Go and Node.js 22. From this directory run `npm install` once, then `hugo server` and open http://localhost:1313/. + +## Where things go + +Documentation pages live under `content/en/docs/`, one markdown file per page with `title`, `weight` and `description` in the front matter; the weight sets the order in the sidebar. The landing page is `content/en/_index.md`. Write one continuous line per paragraph. diff --git a/site/assets/icons/logo.svg b/site/assets/icons/logo.svg new file mode 100644 index 00000000..1e4fb76a --- /dev/null +++ b/site/assets/icons/logo.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/site/assets/scss/_styles_project.scss b/site/assets/scss/_styles_project.scss new file mode 100644 index 00000000..d57f3c9f --- /dev/null +++ b/site/assets/scss/_styles_project.scss @@ -0,0 +1,436 @@ +// The logo is the full wordmark, so the site name next to it is dropped. +.navbar-brand__name { + display: none; +} + +.navbar-brand__logo svg { + height: 1.75rem; + width: auto; +} + +// Landing page +// +// The sections below use Bootstrap's colour-mode variables rather than +// Docsy's fixed box colours, so the light and dark themes both read as one +// page. The cover keeps the brand navy in both. + +$bs-accent: #93a4ff; + +.td-home { + .td-cover-block { + background: radial-gradient(ellipse at 15% 0%, lighten($primary, 12%), transparent 55%), + radial-gradient(ellipse at 100% 100%, darken($secondary, 10%), transparent 50%), + $primary; + } + + .td-cover-block .td-overlay__inner { + padding-top: 2rem; + padding-bottom: 2.5rem; + } +} + +.cover-hero { + .eyebrow { + display: inline-block; + margin-bottom: 1rem; + padding: 0.25rem 0.75rem; + border: 1px solid rgba(255, 255, 255, 0.35); + border-radius: 2rem; + color: rgba(255, 255, 255, 0.85); + font-size: 0.85rem; + letter-spacing: 0.02em; + + a { + color: #fff; + text-decoration: underline; + text-decoration-color: rgba(255, 255, 255, 0.5); + } + } + + h1 { + font-size: clamp(2.2rem, 4.5vw, 3.6rem); + font-weight: 600; + line-height: 1.1; + letter-spacing: -0.01em; + } + + .lead { + max-width: 34rem; + color: rgba(255, 255, 255, 0.85); + + code { + padding: 0.1em 0.35em; + border-radius: 0.25rem; + background: rgba(255, 255, 255, 0.12); + color: #fff; + } + } + + .hero-actions .btn { + padding: 0.6rem 1.25rem; + } + + .btn-accent { + background-color: $bs-accent; + border-color: $bs-accent; + color: $primary; + font-weight: 600; + + &:hover, + &:focus { + background-color: lighten($bs-accent, 6%); + border-color: lighten($bs-accent, 6%); + color: $primary; + } + } + + .hero-facts { + margin: 1.5rem 0 0; + padding: 0; + list-style: none; + display: flex; + flex-wrap: wrap; + gap: 0.5rem 1.5rem; + color: rgba(255, 255, 255, 0.75); + font-size: 0.95rem; + + li::before { + content: ""; + display: inline-block; + width: 0.45rem; + height: 0.45rem; + margin-right: 0.5rem; + border-radius: 50%; + background: $bs-accent; + vertical-align: middle; + } + } +} + +// The replication figure on the cover. It sits on the navy background, so +// it is drawn in light strokes in both themes. +.replicas { + margin: 1rem auto 0; + max-width: 30rem; + + svg { + display: block; + width: 100%; + font-family: $font-family-monospace; + } + + text { + fill: #fff; + font-size: 13px; + } + + .sub { + fill: rgba(255, 255, 255, 0.7); + font-size: 12px; + } + + .crd rect { + fill: rgba(255, 255, 255, 0.08); + stroke: #fff; + stroke-width: 1.5; + } + + .node > rect:first-child { + fill: rgba(255, 255, 255, 0.06); + stroke: rgba(255, 255, 255, 0.35); + stroke-width: 1.5; + } + + .blk { + fill: $bs-accent; + } + + .links path { + fill: none; + stroke: rgba(255, 255, 255, 0.6); + stroke-width: 1.5; + } + + .sync path { + fill: none; + stroke: $bs-accent; + stroke-width: 2; + stroke-dasharray: 4 4; + } + + figcaption { + margin-top: 0.75rem; + color: rgba(255, 255, 255, 0.7); + font-size: 0.9rem; + } +} + +@media (prefers-reduced-motion: no-preference) { + .replicas .sync path { + animation: replicas-sync 1.6s linear infinite; + } +} + +@keyframes replicas-sync { + to { + stroke-dashoffset: -16; + } +} + +// Plain sections, both themes. +.bs-section { + padding: 4rem 0; + background: var(--bs-body-bg); + color: var(--bs-body-color); + + &--tint { + background: var(--bs-tertiary-bg); + } + + &--border { + border-top: 1px solid var(--bs-border-color); + } + + h2 { + font-weight: 600; + margin-bottom: 0.5rem; + } + + .section-lead { + max-width: 44rem; + color: var(--bs-secondary-color); + margin-bottom: 2rem; + } +} + +// The lead paragraph under the cover. +.bs-intro { + font-size: 1.2rem; + max-width: 50rem; + margin: 0 auto; + text-align: center; +} + +// Feature cards. +.bs-cards { + display: grid; + gap: 1.25rem; + grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr)); +} + +.bs-card { + display: flex; + flex-direction: column; + padding: 1.5rem; + border: 1px solid var(--bs-border-color); + border-radius: 0.75rem; + background: var(--bs-body-bg); + color: inherit; + text-decoration: none; + transition: border-color 0.15s ease, transform 0.15s ease; + + .bs-card__icon { + display: inline-flex; + align-items: center; + justify-content: center; + width: 2.5rem; + height: 2.5rem; + margin-bottom: 1rem; + border-radius: 0.5rem; + background: rgba($bs-accent, 0.18); + color: $secondary; + font-size: 1.1rem; + } + + h3 { + font-size: 1.15rem; + font-weight: 600; + margin-bottom: 0.5rem; + + a { + color: inherit; + text-decoration: none; + } + } + + p { + margin-bottom: 0; + color: var(--bs-secondary-color); + } + + code { + font-size: 0.85em; + } + + .bs-card__more { + margin-top: auto; + padding-top: 1rem; + font-weight: 600; + color: var(--bs-link-color); + } +} + +a.bs-card:hover, +a.bs-card:focus { + border-color: $secondary; + transform: translateY(-2px); + text-decoration: none; + color: inherit; +} + +[data-bs-theme="dark"] { + .bs-card .bs-card__icon { + color: $bs-accent; + } + + a.bs-card:hover, + a.bs-card:focus { + border-color: $bs-accent; + } +} + +// The terminal block: three commands to a replicated volume. +.bs-terminal { + max-width: 44rem; + margin: 0 auto; + border-radius: 0.75rem; + overflow: hidden; + background: #16213a; + color: #e6e9f2; + box-shadow: 0 1rem 2.5rem rgba(0, 0, 0, 0.25); + + .bs-terminal__bar { + display: flex; + align-items: center; + gap: 0.4rem; + padding: 0.6rem 1rem; + background: #0f172c; + font-size: 0.8rem; + color: rgba(255, 255, 255, 0.5); + + span { + width: 0.7rem; + height: 0.7rem; + border-radius: 50%; + background: rgba(255, 255, 255, 0.25); + } + + em { + margin-left: 0.5rem; + font-style: normal; + } + } + + pre { + margin: 0; + padding: 1.25rem 1.5rem; + background: transparent; + color: inherit; + font-size: 0.9rem; + line-height: 1.7; + overflow-x: auto; + } + + .c { + color: rgba(255, 255, 255, 0.45); + } + + .p { + color: $bs-accent; + user-select: none; + } + + // The site-wide click-to-copy button is for documentation code blocks. + .td-click-to-copy { + display: none; + } +} + +// Steps beside the terminal. +.bs-steps { + margin: 0; + padding: 0; + list-style: none; + counter-reset: step; + + li { + position: relative; + padding-left: 2.75rem; + margin-bottom: 1.25rem; + counter-increment: step; + + &::before { + content: counter(step); + position: absolute; + left: 0; + top: 0.1rem; + width: 2rem; + height: 2rem; + border-radius: 50%; + background: rgba($bs-accent, 0.18); + color: $secondary; + font-weight: 600; + text-align: center; + line-height: 2rem; + } + + strong { + display: block; + } + + span { + color: var(--bs-secondary-color); + } + } +} + +[data-bs-theme="dark"] .bs-steps li::before { + color: $bs-accent; +} + +// Cozystack and CNCF. +.cncf-block { + text-align: center; + + .cncf-logo { + width: 50%; + max-width: 30rem; + height: auto; + margin-top: 1rem; + } + + // Docsy sets data-bs-theme on ; the white CNCF logo replaces the + // colour one on the dark background. + .cncf-logo--light { + display: inline-block; + } + + .cncf-logo--dark { + display: none; + } +} + +[data-bs-theme="dark"] .cncf-block { + .cncf-logo--light { + display: none; + } + + .cncf-logo--dark { + display: inline-block; + } +} + +.bs-ack { + font-size: 0.95rem; + color: var(--bs-secondary-color); + max-width: 50rem; + margin: 0 auto; +} + +// Documentation landing: the track cards above the section index. +.td-content .bs-cards { + margin: 1.5rem 0 2.5rem; +} + +// Code blocks follow the light and dark theme (needs highlight.noClasses: false). +@import "td/code-dark"; diff --git a/site/assets/scss/_variables_project.scss b/site/assets/scss/_variables_project.scss new file mode 100644 index 00000000..beb25a96 --- /dev/null +++ b/site/assets/scss/_variables_project.scss @@ -0,0 +1,3 @@ +// Brand colour taken from the logo. +$primary: #1b2a4a; +$secondary: #3b5b8c; diff --git a/site/content/en/_index.md b/site/content/en/_index.md new file mode 100644 index 00000000..ed6b384a --- /dev/null +++ b/site/content/en/_index.md @@ -0,0 +1,163 @@ +--- +title: Blockstor +--- + +
+
+
+
+
+

Early-stage project, developed as part of Cozystack

+

Replicated block storage, run by Kubernetes

+

Blockstor turns the local disks of your nodes into DRBD-replicated volumes on LVM or ZFS. The whole cluster state lives in custom resources, so you operate storage the way you operate everything else: with kubectl, GitOps and a CLI.

+
+Quickstart +Documentation +GitHub +
+
    +
  • LINSTOR-compatible API
    • +
  • LVM, LVM-thin, ZFS
    • +
  • DRBD 9 replication
    • +
  • amd64 and arm64
    • +
+
+
+{{< replicas >}} +
+
+
+
+
+ +
+
+
+

Blockstor speaks the LINSTOR REST API, so the clients you already run keep working unchanged: linstor-csi, piraeus-operator, ha-controller, golinstor and the linstor command. What changes is underneath. There is no database to back up, no in-memory state to lose on a restart, and no controller polling nodes.

+
+
+
+ +
+
+
+

From a pool to a replicated volume

+

Register the storage once, describe the placement policy once, and every volume spawned from the group gets the same replication.

+
+
+
+
blockstor
+
# a thin pool on each node becomes a storage pool named "data"
+$ blockstor storage-pool create lvmthin worker-1 data my_vg/my_thinpool
+
+# the group holds the policy: two replicas, placed from "data"
+$ blockstor resource-group create replicated --place-count 2 --storage-pool data
+$ blockstor volume-group create replicated
+
+# one line per volume
+$ blockstor resource-group spawn replicated my-volume 10G
+$ blockstor resource list
+
+
+
+
    +
  1. Register the storageA storage pool points at an LVM volume group, a thin pool or a ZFS pool on a node. Blockstor can also build one from a raw disk.
    2. +
  2. Describe the policyA resource group says how many replicas a volume gets, from which pools, and how they spread across nodes and zones.
    3. +
  3. Spawn volumes, or let CSI do itEach spawn creates a definition that the controller places and the satellites bring up as DRBD replicas. A StorageClass does the same for every PersistentVolumeClaim.
    4. +
+

Run it yourself in the quickstart

+
+
+
+
+
+ +
+
+
+

Why blockstor

+

The same storage stack as LINSTOR and Piraeus, with the control plane rebuilt on Kubernetes primitives.

+
+
+
+

Kubernetes-native state

+

Nodes, pools, definitions, replicas and snapshots are custom resources with printer columns. Each satellite watches the slice for its own node and reports what it sees through Status. Back it up, diff it and drive it from GitOps like any other object.

+
+
+
+

Drop-in for the LINSTOR ecosystem

+

The REST API matches the LINSTOR wire shape, verified against golinstor. CSI, the HA controller and existing runbooks carry over. A running LINSTOR cluster can be taken over in place, without copying data.

+
+
+
+

Small footprint

+

Three statically linked Go services and a kernel module. No JVM, no external database, no controller-to-node protocol. Multi-arch images for linux/amd64 and linux/arm64 fit edge clusters and small ARM nodes.

+
+
+
+

The features you expect

+

Replicas on LVM, LVM-thin, ZFS and file backends. Automatic placement with zones and node constraints, tiebreakers and quorum, LUKS encryption, online resize, snapshots, clones and restores, and mutual TLS on the API.

+
+
+
+
+
+ +
+
+
+

Find your way

+

Pick the page that matches what you are doing.

+
+ +
+

Quickstart

+

From an empty cluster to a replicated PersistentVolumeClaim.

+Start here + + +
+

Installation

+

Requirements, the manifests, registering nodes and storage pools.

+Install + + +
+

Architecture

+

How the apiserver, the controller and the satellites share the work.

+Understand + + +
+

Coming from LINSTOR

+

Take over an existing cluster in place and what differs afterwards.

+Migrate + + +
+

Using the CLI

+

Nodes, pools, volumes and snapshots from the blockstor command.

+Operate + + +
+

Developer guide

+

Build it, run the test tiers, and find your way around the code.

+Contribute + +
+
+
+
+ +{{< cncf >}} + +
+
+
+

Acknowledgements

+

Blockstor was inspired by LINBIT's LINSTOR, and it operates DRBD to provide block-level replication. It deliberately speaks LINSTOR's API so that the ecosystem the LINSTOR community has built keeps working unchanged. Heartfelt thanks to LINBIT and to the wider DRBD, LINSTOR and Piraeus community.

+

Blockstor is licensed under Apache 2.0. LINSTOR, LINBIT and DRBD are trademarks or registered trademarks of LINBIT. Blockstor is an independent project and is not affiliated with, endorsed by, or sponsored by LINBIT.

+
+
+
diff --git a/site/content/en/developer-guide/_index.md b/site/content/en/developer-guide/_index.md new file mode 100644 index 00000000..ae25ed3f --- /dev/null +++ b/site/content/en/developer-guide/_index.md @@ -0,0 +1,21 @@ +--- +title: "Developer guide" +linkTitle: "Developer guide" +weight: 30 +menu: + main: + weight: 20 +description: "Build blockstor, run its tests, and find your way around the code." +cascade: + type: docs +--- + +This section is for people who work on blockstor itself. If you want to run blockstor, the [Documentation](/docs/) is the place to start. + +- [Building](building/): binaries, images and the local dev stand. +- [Testing](testing/): the test tiers, from unit tests to the operator CLI matrix. +- [Repository layout](repository/): where each part of the code lives. +- [Internals](internals/): the rules the code relies on, with references to it. +- [Contributing](contributing/): licensing rules, commits and pull requests. + +Design notes and audits that are useful background but not part of this guide live in the repository's [`docs/`](https://github.com/cozystack/blockstor/tree/main/docs) directory. diff --git a/site/content/en/developer-guide/building.md b/site/content/en/developer-guide/building.md new file mode 100644 index 00000000..5aa44436 --- /dev/null +++ b/site/content/en/developer-guide/building.md @@ -0,0 +1,69 @@ +--- +title: "Building" +linkTitle: "Building" +weight: 10 +description: "Build the binaries and images, and bring up the dev stand." +--- + +## Binaries + +Blockstor is a Go module, `github.com/cozystack/blockstor`. `make build` regenerates the manifests and deep-copy code, runs `go fmt` and `go vet`, and builds the controller, the satellite, the CLI and the migration tool into `bin/`. The API server builds the usual Go way: + +```sh +make build +go build -o bin/apiserver ./cmd/apiserver +``` + +| Binary | Source | What it is | +|---|---|---| +| `controller` | `cmd/controller` | controller-runtime manager with the controller-side reconcilers. | +| `apiserver` | `cmd/apiserver` | the stateless LINSTOR-compatible REST front end. | +| `satellite` | `cmd/satellite` | the per-node manager that drives DRBD, LUKS and storage. | +| `blockstor` | `cmd/blockstor` | the operator CLI. | +| `linstor-migrate` | `cmd/linstor-migrate` | the LINSTOR database converter. | + +Other generators: `make manifests` regenerates the CRDs and RBAC from the kubebuilder markers in `api/v1alpha1`, and `make generate` regenerates the deep-copy code. + +## Images + +One multi-stage [`Dockerfile`](https://github.com/cozystack/blockstor/blob/main/Dockerfile) builds all three images: the `controller`, `apiserver` and `satellite` targets. The controller and API server images are distroless; the satellite image is Debian-based, because it shells out to `drbdadm`, `lvs`, `zfs` and `cryptsetup`. + +Releases are built by [`release.yml`](https://github.com/cozystack/blockstor/blob/main/.github/workflows/release.yml) on a `v*` tag and pushed to `ghcr.io/cozystack/blockstor-{controller,apiserver,satellite}` for `linux/amd64` and `linux/arm64`. + +## The dev stand + +`stand/` brings up a throwaway Talos cluster in QEMU, with DRBD, ZFS and LVM in the node image, so you can exercise the real data path on your machine. + +Host requirements: + +- Linux x86_64 with KVM (`/dev/kvm` accessible); +- `talosctl`, `kubectl`, `helm`, `qemu-system-x86_64`; +- the DRBD 9 kernel module loaded on the host; +- about 8 GB of free RAM and 20 GB of disk per cluster. + +```sh +make build-images # build the images and push them to the local registry +make up # create the cluster (default name "blockstor") +make blockstor # install the CRDs, controller, API server and satellites +make piraeus # linstor-csi in external-controller mode, after blockstor +make smoke-blockstor # quick end-to-end check +make down +``` + +Real-disk pools on extra disks: + +```sh +make pools TYPE=zfs # TYPE is zfs, lvm or both (the default) +``` + +`make smoke-blockstor` always uses the stand's own `stand` pool. `make burnin-blockstor` takes the pool to churn as `POOL=`. + +Several clusters can run side by side, each with its own `10..0.0/24` network. Their configuration lands in `.work//`: + +```sh +make up NAME=alice +eval "$(make use NAME=alice)" +kubectl get nodes +``` + +The stand pins every image by digest, so each rebuild rolls the pods. diff --git a/site/content/en/developer-guide/contributing.md b/site/content/en/developer-guide/contributing.md new file mode 100644 index 00000000..d35e8650 --- /dev/null +++ b/site/content/en/developer-guide/contributing.md @@ -0,0 +1,29 @@ +--- +title: "Contributing" +linkTitle: "Contributing" +weight: 50 +description: "Licensing rules, and what a change needs before it merges." +--- + +## Licensing: Apache 2.0, clean room + +Blockstor is Apache 2.0. Do not copy structure or code from GPL sources, in particular `linstor-server` and `drbd-utils`. You may read them to understand behaviour, never to reproduce how they are written. + +Hard rules: + +- No GPL-licensed spec files in the tree. LINSTOR's `rest_v1_openapi.yaml` declares GPLv3, and code generated from it is a derivative work. +- No code generated from a GPL source by any tool. +- No Go dependency under GPL, AGPL, LGPL or SSPL. CI enforces this through [`license-check.yml`](https://github.com/cozystack/blockstor/blob/main/.github/workflows/license-check.yml), which runs `go-licenses check --disallowed_types=forbidden,restricted` over `./cmd/...`. That is a denylist: it fails on licenses `go-licenses` classifies as forbidden or restricted, and lets everything else through, so check a new dependency's license yourself. +- Every Go source file carries an `SPDX-License-Identifier: Apache-2.0` header. + +Apache-2.0 projects that are safe to study and depend on: [`golinstor`](https://github.com/LINBIT/golinstor), the authoritative description of the REST wire shape; [`piraeus-operator`](https://github.com/piraeusdatastore/piraeus-operator); [`linstor-csi`](https://github.com/piraeusdatastore/linstor-csi); [`drbd-reactor`](https://github.com/LINBIT/drbd-reactor); and the public DRBD 9 documentation and man pages. + +When in doubt, work clean room: one person reads the GPL source and writes a behavioural spec, and another implements it without seeing the original. + +## Before you open a pull request + +- `go test ./...` passes and `make lint` is clean. +- A bug fix comes with a test at the lowest tier that catches it. A bug in the operator CLI also gets a cell in `tests/e2e/cli-matrix/`. +- A new CLI verb or a change to a wire shape is checked with the parity refresh, and an intended difference from upstream gets a row in [`docs/cli-parity-known-deltas.md`](https://github.com/cozystack/blockstor/blob/main/docs/cli-parity-known-deltas.md). +- User-facing behaviour is documented on this site, under `site/content/en/docs/`. +- Commits follow Conventional Commits and are signed off (`git commit --signoff`). diff --git a/site/content/en/developer-guide/internals.md b/site/content/en/developer-guide/internals.md new file mode 100644 index 00000000..159a8a53 --- /dev/null +++ b/site/content/en/developer-guide/internals.md @@ -0,0 +1,77 @@ +--- +title: "Internals" +linkTitle: "Internals" +weight: 40 +description: "The rules the code relies on, with pointers into it." +--- + +This page is about the invariants whose violation would quietly corrupt cluster state. For the user-level picture, see [Architecture](/docs/architecture/). For a tour of the code, follow the call graph from `cmd/controller/main.go` and `cmd/satellite/main.go`. + +## Spec and Status + +Every blockstor CRD has two halves: + +- **Spec** is desired state. Users, REST handlers and the controller's placement write it. The satellite writes Spec in two places: `Resource.Spec.Flags` (and `ToggleDiskCancel`) while it carries out a toggle-disk, and the discovered volume groups and ZFS pools in `Node.Spec.Props`. +- **Status** is observed state. The satellite writes what it reads from the kernel (`drbdsetup events2`, `drbdadm status`, `lvs`), and the controller's allocators write values derived from Spec. The REST API writes Status in one place: the resource activate endpoint with `?reallocate-port=true` clears `Resource.Status.DRBDPort` through a Status patch. The allocator only fills an empty `Spec.DRBDPort`, and the controller copies the spec value back into `Status`, so the port does not change. + +A whole-object `Update` is unsafe whenever both halves have different writers: a Spec write in flight would overwrite a concurrent Status write, and the other way round. Status goes through the Status subresource. Fields with more than one writer, such as `Resource.Status.DRBDPort` from the controller next to `DiskState` and `CurrentGi` from the satellite, go through Server-Side Apply with separate field managers, `blockstor-controller` and `blockstor-satellite`, so each side owns only its own fields. + +| Field | Half | Why | +|---|---|---| +| `Resource.Spec.NodeName` | Spec | Placement target. | +| `Resource.Spec.Flags` | Spec | `DISKLESS` and `TIE_BREAKER` are requested, not observed. | +| `Resource.Spec.StoragePool` | Spec | Written by the allocator. | +| `Resource.Spec.Volumes[i].SeedFromGi` | Spec | DRBD generation ID to stamp on first activation. | +| `Resource.Status.DrbdState`, `InUse` | Status | Kernel state from `events2`. | +| `Resource.Status.Volumes[i].DiskState`, `CurrentGi` | Status | Per-volume kernel state. | +| `Node.Status.ConnectionStatus` | Status | Whether the satellite is alive. | + +A field that fits neither half, typically a transient hint, goes into an annotation. + +## Writes are patches + +Every mutating path, in the REST handlers and in the CLI, goes through the store's `Patch*` entry points. They fetch the current object, apply the caller's change to it, and retry on conflict. A caller hands over a change, never a whole object, so a key another writer added in between is kept. Checks that depend on current state, such as refusing a shrink, run inside the patch against the state the write lands on. + +Invariants that must hold for every writer live in the CRD, not in a client: size bounds are `Minimum` and `Maximum` on the field, and the controller-allocated identities (`drbdPort`, `drbdNodeID`, `skipInitialSync`) are settable-once CEL rules. A check that lives in one client is not a check the data is subject to. + +## Option inheritance + +DRBD configuration resolves from the broadest scope to the narrowest: + +``` +ControllerConfig -> ResourceGroup -> ResourceDefinition -> Resource +``` + +The typed resolver is `ResolveDRBDOptions` in `pkg/drbd/typed_resolver.go`. Pointer fields carry the nil-versus-set distinction: nil inherits, and any non-nil value, including `false` and zero, overrides. A merge written as `if *src.X { out.X = src.X }` would drop an explicit `false`; `pkg/drbd/typed_resolver_test.go` pins this. + +## Wire shape and CRD shape + +Two shapes coexist: + +1. The wire shape in `pkg/api/v1`, identical to upstream LINSTOR's REST API, with property bags as `map[string]string`. +2. The CRD shape in `api/v1alpha1`, with typed `Spec.DRBDOptions` and `Spec.ExtraProps` for keys not typed yet. + +`pkg/store/k8s/` is the boundary. `drbd_transcode.go` parses wire properties into typed fields on write and emits them back on read, so `golinstor` sees the shape it expects. Unknown `DrbdOptions/*` keys round-trip through `ExtraProps`. + +## Skipping the initial sync + +Adding a third replica to a two-replica volume would otherwise copy the whole device. The skip pipeline: + +1. The satellite's `events2` observer parses `current-uuid` and publishes it as `Resource.Status.Volumes[i].CurrentGi`. +2. When it allocates a new replica, the controller picks the lowest-named `UpToDate` peer's `CurrentGi` and stamps it on the new replica's `Spec.Volumes[i].SeedFromGi`. +3. On first activation the satellite runs `drbdmeta ... set-gi` between `drbdadm create-md` and `drbdadm adjust`, so the first handshake sees the new replica as already in sync. + +`tests/e2e/replica-add-no-resync.sh` gates it end to end. Promoting a tiebreaker to a replica with data deliberately does a full sync instead; see row 84 of the [parity deltas](https://github.com/cozystack/blockstor/blob/main/docs/cli-parity-known-deltas.md). + +## Reconcilers + +Controller side, in `internal/controller/`: + +- `ResourceReconciler` allocates the DRBD node ID, port and minor, picks `SeedFromGi`, and promotes a diskless replica that is in active use. +- `ResourceDefinitionReconciler` adds a diskless tiebreaker when a definition has an even number of replicas with data, and sets the quorum policy. +- `NodeReconciler`, `ResourceGroupReconciler`, `StoragePoolReconciler` and `SnapshotReconciler` own the invariants of their kinds. +- Cross-cutting controllers: the node heartbeat watchdog, node label sync, auto-diskful, auto-evict, auto-snapshot, group rebalancing and replica migration. + +Satellite side, in `pkg/satellite/controllers/`: each satellite runs controller-runtime reconcilers that watch only the objects whose `spec.nodeName` is its own node, drive the DRBD, LUKS and storage layers, and write observed state back through Status. `pkg/dispatcher/` translates the CRDs into the desired state they apply. + +There is no satellite to controller connection. Both sides talk only to the Kubernetes API. diff --git a/site/content/en/developer-guide/repository.md b/site/content/en/developer-guide/repository.md new file mode 100644 index 00000000..ce820156 --- /dev/null +++ b/site/content/en/developer-guide/repository.md @@ -0,0 +1,36 @@ +--- +title: "Repository layout" +linkTitle: "Repository layout" +weight: 30 +description: "Where each part of blockstor lives." +--- + +``` +api/v1alpha1/ CRD types and kubebuilder markers +cmd/ controller, apiserver, satellite, blockstor CLI, linstor-migrate +internal/controller/ controller-side reconcilers +internal/cli/ the blockstor CLI +pkg/ REST, stores, satellite, storage, DRBD, LUKS, placer, dispatcher +config/ generated CRDs, RBAC and kustomize bases +stand/ the Talos and QEMU dev stand and the reference manifests +tests/ contract, integration, e2e, cli-matrix and replay suites +docs/ design notes, parity deltas and audit history +site/ this website +``` + +| Path | What lives there | +|---|---| +| `api/v1alpha1/` | The CRD types. `make manifests` turns their markers into `config/crd/bases/`. | +| `pkg/api/v1/` | The REST wire types, shaped like upstream LINSTOR's, and the layer-stack resolver. | +| `pkg/rest/` | The LINSTOR-compatible REST handlers. | +| `pkg/store/`, `pkg/store/k8s/` | The in-memory and CRD-backed stores behind one `store.Store` interface, and the translation between wire and CRD shapes. | +| `internal/controller/` | Controller-side reconcilers: resources, definitions, groups, pools, snapshots, nodes, plus tiebreakers, auto-diskful, eviction, rebalancing and migration. | +| `pkg/satellite/controllers/` | Satellite-side reconcilers for resources, pools, snapshots and physical devices, and the `events2` observer. | +| `pkg/satellite/` | The DRBD, LUKS and storage layer reconciler and snapshot shipping. | +| `pkg/storage/` | Storage providers: LVM, LVM-thin, ZFS, ZFS-thin, loopfile and file. | +| `pkg/drbd/` | `drbdadm` and `drbdsetup` wrappers, the `.res` renderer, the `events2` parser and the options resolver. | +| `pkg/luks/` | The `cryptsetup` wrapper. | +| `pkg/placer/` | The autoplacer: capacity-weighted, anti-affinity aware. | +| `pkg/dispatcher/` | Translates CRDs into the desired state a satellite applies: layers, options, passphrases. | +| `internal/cli/` | The `blockstor` CLI. Its design is in [`docs/cli-design.md`](https://github.com/cozystack/blockstor/blob/main/docs/cli-design.md). | +| `cmd/linstor-migrate/` | The LINSTOR database converter. | diff --git a/site/content/en/developer-guide/testing.md b/site/content/en/developer-guide/testing.md new file mode 100644 index 00000000..40c02d23 --- /dev/null +++ b/site/content/en/developer-guide/testing.md @@ -0,0 +1,64 @@ +--- +title: "Testing" +linkTitle: "Testing" +weight: 20 +description: "The test tiers and how to run each of them." +--- + +Each tier answers a different question, and a change is expected to come with tests at the lowest tier that can catch the bug. + +| Tier | What it covers | Where | +|---|---|---| +| L1 unit | Logic, run on every commit | `*_test.go` next to the code | +| L2 contract, golden | Recorded `golinstor` responses replayed against the server, byte diff | `tests/contract/` | +| L3 contract, oracle | The same requests to a real LINSTOR and to blockstor, JSON diff | `tests/contract/` | +| Integration | Controllers and REST against a real Kubernetes API server (envtest), satellites faked | `tests/integration/` | +| L4 integration on DRBD | `make smoke` on the stand | `tests/smoke*.sh` | +| L5 e2e | csi-sanity and piraeus-operator e2e on the stand | `tests/e2e/` | +| L6 operator CLI | The real CLI, through REST and the satellite, down to the DRBD kernel state | `tests/e2e/cli-matrix/` | +| L7 replay | Operator workflows replayed, and the CLI parity diff against upstream | `tests/operator-harness/` | + +## Unit tests and lint + +```sh +go test ./... +make lint +``` + +Tests that need envtest skip themselves when `KUBEBUILDER_ASSETS` is not set. `make test` downloads the envtest binaries and runs the suite with them. `make lint` must be clean before a push. + +## Integration tests + +The integration suite boots an in-process Kubernetes API server and etcd with envtest, applies the CRDs, and runs every reconciler plus the REST server against it. Satellites are replaced with fakes that write Status the way a real satellite would. The tests carry the `integration` build tag: + +```sh +make setup-envtest +go test -tags integration ./tests/integration/... +``` + +## e2e on the stand + +```sh +make e2e-list +make e2e NAME=alice SCENARIO=tiebreaker +``` + +Each scenario is a shell script in `tests/e2e/`. New scenarios register their cleanup with `register_strict_cleanup` from `tests/e2e/lib.sh`: it deletes the named definitions, waits for terminating satellite pods, and checks that the cluster is back to its baseline. A scenario that leaves the cluster dirty is turned into a failure, so the next one is not blamed for it. + +## The operator CLI matrix + +`tests/e2e/cli-matrix/` holds one script per operator-reported CLI bug. A cell passes only when the replica's Status, written by the satellite's observer, and a `drbdsetup status` probe on the node both reach the expected state. A `200 OK` from the API is not enough. See the [cell README](https://github.com/cozystack/blockstor/blob/main/tests/e2e/cli-matrix/README.md). + +## Replay and parity + +```sh +# One replay workflow +BS_URL=http://127.0.0.1:3370 \ + tests/operator-harness/replay-runner.sh tests/operator-harness/replay/pvc-lifecycle.yaml + +# CLI parity against an upstream LINSTOR +BS_URL=http://127.0.0.1:3370 UP_URL=http://127.0.0.1:3371 \ + tests/operator-harness/cli-parity-refresh.sh /tmp/cli-parity +``` + +You set up the port-forwards yourself. The parity run fails on any difference from upstream that is not listed in [`docs/cli-parity-known-deltas.md`](https://github.com/cozystack/blockstor/blob/main/docs/cli-parity-known-deltas.md). Add a row there only for an intended difference; a regression gets fixed instead. diff --git a/site/content/en/docs/_index.md b/site/content/en/docs/_index.md new file mode 100644 index 00000000..dbb006c7 --- /dev/null +++ b/site/content/en/docs/_index.md @@ -0,0 +1,42 @@ +--- +title: "Documentation" +linkTitle: "Docs" +weight: 20 +menu: + main: + weight: 10 +description: "Install, operate and understand blockstor." +--- + +Blockstor is a Kubernetes control plane for LVM and ZFS storage with DRBD replication. It speaks a LINSTOR-compatible REST API, so `linstor-csi`, `piraeus-operator`, `ha-controller` and `golinstor` keep working unchanged, and it ships its own `blockstor` command-line tool. + +
+
+
+

New here

+

The Quickstart takes an empty cluster to a replicated PersistentVolumeClaim. Then read the Overview for what works today and what does not.

+Quickstart +
+
+
+

Setting up a cluster

+

Installation covers requirements, the manifests, nodes and pools. CSI connects it to StorageClasses, Networking lists the ports.

+Installation +
+
+
+

Running it day to day

+

Operations for node maintenance, resize, snapshots and upgrades, Troubleshooting when something is off, and the CLI reference.

+Operations +
+
+
+

Coming from LINSTOR

+

Migrating from LINSTOR takes over a cluster in place. Compatibility lists what the API answers differently.

+Migration +
+
+ +To understand how it works, read [Architecture](architecture/), [Resources](resources/) and [Layer stack](layer-stack/). If you want to work on blockstor itself, see the [Developer guide](/developer-guide/). + +## All pages diff --git a/site/content/en/docs/architecture.md b/site/content/en/docs/architecture.md new file mode 100644 index 00000000..9d82f672 --- /dev/null +++ b/site/content/en/docs/architecture.md @@ -0,0 +1,144 @@ +--- +title: "Architecture" +linkTitle: "Architecture" +weight: 30 +description: "How blockstor turns a volume request into replicas on disk." +--- + +Blockstor is built like a Kubernetes operator. The desired state of every node, pool and volume is a custom resource, and controllers keep working until the cluster matches it. Nothing important lives only in a process's memory, so any blockstor pod can restart without losing state. + +## Components + +```mermaid +flowchart TB + subgraph clients["Clients"] + csi["linstor-csi"] + other["piraeus-operator, ha-controller, golinstor"] + cli["blockstor CLI"] + end + + subgraph k8s["Kubernetes API"] + crds[("blockstor custom resources")] + end + + subgraph cp["blockstor-system"] + api["blockstor-apiserver
LINSTOR-compatible REST, mTLS"] + ctrl["blockstor-controller
placement and allocation"] + end + + subgraph nodes["Storage nodes"] + sat1["satellite on node 1"] + sat2["satellite on node 2"] + sat3["satellite on node 3"] + end + + csi -->|REST| api + other -->|REST| api + cli -->|kubeconfig| crds + api <--> crds + ctrl <--> crds + crds <--> sat1 + crds <--> sat2 + crds <--> sat3 +``` + +- **blockstor-apiserver** serves the LINSTOR REST API over mutual TLS. It does not keep state of its own: every request becomes a read or a write of custom resources. It runs as several replicas behind one Service. +- **blockstor-controller** watches the custom resources and fills in what users do not choose: which nodes get replicas, the DRBD port, minor numbers and node IDs, and tiebreakers and quorum settings. +- **blockstor-satellite** runs on every storage node. It creates the logical volumes or zvols, sets up LUKS and DRBD, and reports what it sees back into the custom resources. +- **The `blockstor` CLI** reads and writes the custom resources directly with your kubeconfig, without going through the REST API. + +There is no direct connection between the controller and the satellites. Everything goes through the Kubernetes API. + +## Custom resources are the source of truth + +Every object you know from LINSTOR is a cluster-scoped custom resource in the `blockstor.cozystack.io` group: + +```mermaid +flowchart LR + RG["ResourceGroup
policy: place count, pool, layers"] + RD["ResourceDefinition
one volume set, its size and layers"] + R1["Resource
replica on node 1"] + R2["Resource
replica on node 2"] + R3["Resource
tiebreaker on node 3"] + SP["StoragePool
per node"] + N["Node"] + S["Snapshot"] + + RG --> RD + RD --> R1 + RD --> R2 + RD --> R3 + R1 -. lives in .-> SP + SP -. on .-> N + RD --> S +``` + +Each resource has two halves: + +- **spec** is what you, the controller or the API asked for. +- **status** is what the satellites observed: disk states, connection states, free capacity. + +The split is a rule of thumb with a few exceptions: a satellite writes a replica's `spec.flags` while it toggles a disk and records the storage it discovered in the node's `spec.props`, and the API clears a replica's DRBD port in `status` when you ask it to reallocate the port. Because the state is ordinary Kubernetes objects, you can read it with `kubectl get resources`, back it up with your usual tools, and drive it from GitOps. + +## How a volume is created + +This is the path from a PersistentVolumeClaim to replicas on disk: + +```mermaid +sequenceDiagram + autonumber + participant PVC as PVC + participant CSI as linstor-csi + participant API as blockstor-apiserver + participant K as Kubernetes API + participant C as blockstor-controller + participant S as satellites + + PVC->>CSI: provision a volume + CSI->>API: create the resource definition and place it + API->>K: write ResourceDefinition and Resources + K-->>C: change seen + C->>K: pick nodes, allocate DRBD port, minors and node IDs, add a tiebreaker + K-->>S: each satellite sees the Resources for its node + S->>S: create LV or zvol, set up LUKS and DRBD + S->>K: report disk and connection state + API-->>CSI: volume is ready + CSI-->>PVC: bound, device /dev/drbdN +``` + +## Each satellite watches only its own node + +A satellite reacts only to the resources assigned to its node: replicas and pools whose node name is its own. Adding nodes adds satellites, and none of them has to talk to a central process. + +```mermaid +flowchart LR + subgraph K["Kubernetes API"] + r1["Resource pvc-1 on node-1"] + r2["Resource pvc-1 on node-2"] + r3["Resource pvc-2 on node-1"] + end + s1["satellite node-1"] + s2["satellite node-2"] + r1 --> s1 + r3 --> s1 + r2 --> s2 +``` + +## Layers on a node + +Every replica is a stack of layers. The satellite builds it from the bottom up when it creates a replica and takes it apart from the top down when it removes one. + +```mermaid +flowchart TB + pod["Pod sees /dev/drbdN"] + drbd["DRBD: replication to the other nodes"] + luks["LUKS: encryption at rest, optional"] + storage["STORAGE: LVM LV, LVM-thin LV, zvol or file"] + pod --> drbd --> luks --> storage +``` + +The default stack is DRBD on top of STORAGE. See [Layer stack](../layer-stack/) for the other combinations. + +## Replicas, tiebreakers and quorum + +With an even number of data replicas, the controller adds a diskless tiebreaker on another node, so DRBD always has a majority to decide which side keeps writing after a network split. A replica without a disk can also serve I/O over the network to a pod on a node that holds no copy of the data. diff --git a/site/content/en/docs/cli.md b/site/content/en/docs/cli.md new file mode 100644 index 00000000..df1a4e02 --- /dev/null +++ b/site/content/en/docs/cli.md @@ -0,0 +1,154 @@ +--- +title: "Using the CLI" +linkTitle: "Using the CLI" +weight: 60 +description: "Manage nodes, pools, volumes and snapshots with the blockstor command." +--- + +`blockstor` is the command-line tool for blockstor. It reads your kubeconfig and works on the blockstor custom resources directly, so it needs no port-forward and no API certificate: if `kubectl` can reach the cluster, so can `blockstor`. + +The commands follow the `linstor` client's noun-verb grammar, with the same short aliases, so runbooks written for LINSTOR mostly carry over by changing the command name. Tools that speak the LINSTOR REST API, such as `linstor-csi`, keep working against the blockstor API server. The examples on this page use `blockstor`. + +## Install + +```sh +go install github.com/cozystack/blockstor/cmd/blockstor@v +``` + +The tool uses the same kubeconfig as `kubectl`: the `KUBECONFIG` variable, then `~/.kube/config`, or the service account when it runs inside a pod. + +## Getting help + +```sh +blockstor --help # every object and action +blockstor resource create --help # usage of one command +``` + +Every object and action has a short alias: `blockstor sp l` is `blockstor storage-pool list`, `blockstor rd c` is `blockstor resource-definition create`, `blockstor r l` is `blockstor resource list`. + +Output options that work on list commands: + +- `-m` or `--machine-readable` prints JSON. +- `--color=auto|always|never` controls colours. They are on only for a terminal, and `NO_COLOR` turns them off. + +## Nodes and storage pools + +```sh +blockstor node create worker-1 10.0.0.11 +blockstor node list + +blockstor physical-storage list +blockstor physical-storage create-device-pool zfs worker-1 /dev/sdb --pool-name data +blockstor storage-pool create lvmthin worker-2 data my_vg/my_thinpool +blockstor storage-pool list +``` + +Take a node out of service. `evacuate` moves its replicas to other nodes; `restore` brings it back: + +```sh +blockstor node evacuate worker-2 +blockstor node restore worker-2 +``` + +## Resource groups + +A resource group holds the policy, and every volume spawned from it gets the same settings: + +```sh +blockstor resource-group create replicated --place-count 2 --storage-pool data +blockstor volume-group create replicated +blockstor resource-group list +``` + +Change the policy later with `blockstor resource-group modify`, for example `--place-count 3`, then apply it to the volumes that already exist with `blockstor resource-group adjust`. + +## Volumes + +Spawn a volume from a group in one step. This creates the definition, its volume and the replicas: + +```sh +blockstor resource-group spawn replicated my-volume 10G +``` + +Or step by step, without a group policy: + +```sh +blockstor resource-definition create my-volume +blockstor volume-definition create my-volume 10G +blockstor resource create my-volume --auto-place 2 --storage-pool data +``` + +To put replicas on named nodes instead of letting blockstor choose: + +```sh +blockstor resource create worker-1 worker-2 my-volume --storage-pool data +``` + +Inspect the state: + +```sh +blockstor resource-definition list +blockstor resource list # replicas, nodes and DRBD state +blockstor resource list --faulty # only replicas that are not healthy +blockstor volume list # sizes and device paths +``` + +A healthy replicated volume shows `UpToDate` on every replica with data, and a `TieBreaker` when there is an even number of them. + +Grow a volume online. Shrinking is refused. `--force` lowers the size in the definition only: the satellite never shrinks a backing device or DRBD, so they keep their size. See [Growing a volume](../operations/#growing-a-volume). + +```sh +blockstor volume-definition set-size my-volume 0 20G +``` + +Turn a diskless replica into one with data, or the other way round: + +```sh +blockstor resource toggle-disk worker-3 my-volume --storage-pool data +blockstor resource toggle-disk worker-3 my-volume --diskless +``` + +Delete a volume with all its replicas: + +```sh +blockstor resource-definition delete my-volume +``` + +## Snapshots + +```sh +blockstor snapshot create my-volume before-upgrade +blockstor snapshot list +``` + +Restore a snapshot as a new volume. This creates the new definition, fills its volumes from the snapshot and places the replicas. The original volume is not touched: + +```sh +blockstor snapshot resource restore \ + --from-resource my-volume --from-snapshot before-upgrade --to-resource my-volume-restored +``` + +`blockstor snapshot resource-definition restore` is another spelling of the same command. To copy a live volume without taking a snapshot yourself, use `blockstor resource-definition clone my-volume my-volume-copy`. + +In-place rollback is not available on purpose: it destroys every snapshot newer than the one you roll back to. Restore into a new volume instead. + +```sh +blockstor snapshot delete my-volume before-upgrade +``` + +## Properties and DRBD options + +Nodes, storage pools, resource groups, volume groups, resource definitions, volume definitions, resources and the controller have `set-property`, `list-properties` and `delete-property`. Volumes, snapshots, physical storage and encryption have no property verbs. Resource definitions, volume definitions and the controller also have `drbd-options`; for a resource group, set DRBD options as `DrbdOptions/...` properties. Settings resolve from the controller to the group to the definition to the replica, and the narrowest one wins. See [Resources](../resources/#how-settings-are-inherited). + +```sh +blockstor resource-group set-property replicated DrbdOptions/Resource/on-no-quorum suspend-io +blockstor resource-definition list-properties my-volume +``` + +## Encryption + +```sh +blockstor encryption create-passphrase '' +``` + +See [Layer stack](../layer-stack/#encryption) before you use it. diff --git a/site/content/en/docs/compatibility.md b/site/content/en/docs/compatibility.md new file mode 100644 index 00000000..28f75e73 --- /dev/null +++ b/site/content/en/docs/compatibility.md @@ -0,0 +1,47 @@ +--- +title: "Compatibility with LINSTOR" +linkTitle: "Compatibility with LINSTOR" +weight: 80 +description: "Where blockstor behaves differently from LINSTOR, and what is not there." +--- + +Blockstor implements the LINSTOR REST API, so `linstor-csi`, `piraeus-operator`, `ha-controller`, `golinstor` and the `linstor` client work against it. This page lists the differences an operator can notice. The full list, kept for the project's parity tests, is in [`docs/cli-parity-known-deltas.md`](https://github.com/cozystack/blockstor/blob/main/docs/cli-parity-known-deltas.md). + +## Not implemented + +| Feature | What happens | +|---|---| +| Backups, backup shipping, remotes such as S3 | The API answers `501 Not Implemented`. Use snapshots and restore inside the cluster. | +| Schedules | The API answers `501 Not Implemented`. | +| `advise` | Not available. | +| In-place snapshot rollback | Refused on purpose, because it destroys every newer snapshot. Restore the snapshot into a new volume instead. | +| Per-connection DRBD peer options (`resource-connection drbd-peer-options`) | Not available. | +| External DRBD metadata (`StorPoolNameDrbdMeta`) | Not supported yet. | +| Other providers: SPDK, NVMe-oF, OpenFlex, Exos | Not supported. | + +## Behaves differently + +| Area | LINSTOR | Blockstor | +|---|---|---| +| Active network interface | `linstor node interface modify --active` switches the satellite's connection to another interface. | Has no effect. DRBD uses the address picked in the order described in [Networking and ports](../networking/#which-address-replication-uses), not an active flag. A change of address reaches DRBD after the node's satellite pod restarts. | +| Default action on lost quorum | `on-no-quorum io-error` | `on-no-quorum suspend-io`: I/O blocks until quorum returns and then resumes, so a replica that was cut off does not get stuck after the network heals. | +| Shrinking a volume | Allowed for stacks that support it. | Refused. `--force` lowers the size in the definition only; the backing devices and DRBD keep their size. | +| DRBD ports and minors | Ports from 7000, minors from 1000. | Ports from 20000 to 20999, minors from 20000. Blockstor can run on the same nodes as LINSTOR without collisions. Change the port range per node with the `DrbdOptions/TcpPortRange` property. | +| Spawning more replicas than there are nodes | Fails and places nothing. | Places as many replicas as it can, reports the shortfall, and adds the rest when more nodes join. | +| Turning a tiebreaker into a replica with data | Can skip the initial sync. | Runs a full sync, to avoid a DRBD split where both sides refuse to connect. | +| Removing a replica below quorum | Removes the quorum options from the definition and prints a note. | Sets `quorum off` explicitly, with the same effect on DRBD. | +| DRBD option values | Validated against the DRBD schema. | Not validated: a wrong value is stored and fails later on the node. Check values before you set them. | +| Autoplacer weights | `MaxFreeSpace` is 1, the other weights 0. | All four weights default to 1. | +| `controller version` | Reports a git commit hash. | Through the REST API, release images report the commit they were built from. A development build that was not given one reports a placeholder instead. | +| `node info` | Shows OS, CPU and memory. | These fields are empty. | + +## Using the `linstor` client + +The `linstor` client works against blockstor's API server. Port-forward the plain HTTP port and point the client at it. That port has no authentication and, unless you fence it with a NetworkPolicy, any pod that can reach the API server's pod IP can use it; see [Networking and ports](../networking/#the-api-server). + +```sh +kubectl -n blockstor-system port-forward deploy/blockstor-apiserver 3370:3370 +linstor --controllers http://localhost:3370 node list +``` + +For everyday work the [`blockstor` CLI](../cli/) is simpler: it needs no port-forward and no certificates, and it accepts the same commands. diff --git a/site/content/en/docs/csi.md b/site/content/en/docs/csi.md new file mode 100644 index 00000000..5d27b644 --- /dev/null +++ b/site/content/en/docs/csi.md @@ -0,0 +1,157 @@ +--- +title: "Provisioning volumes with CSI" +linkTitle: "CSI and StorageClasses" +weight: 25 +description: "Connect linstor-csi to blockstor and provision PersistentVolumes." +--- + +Kubernetes provisions blockstor volumes through the LINSTOR CSI driver, `linstor-csi`. The driver speaks the LINSTOR REST API, and blockstor serves that API, so you point the driver at the blockstor API server instead of a LINSTOR controller. + +## What to deploy + +Piraeus normally ships a whole LINSTOR distribution. With blockstor you take only the client side: + +| Piraeus component | With blockstor | +|---|---| +| `linstor-controller` | Skip. The blockstor controller and API server replace it. | +| `linstor-satellite` | Skip. Blockstor runs its own satellites. | +| `linstor-csi` (controller and node plugin) | Deploy, pointed at the blockstor API server. | +| HA controller, `drbd-reactor` | Optional. Point them at the API server the same way as CSI. | + +## Point the driver at blockstor + +The API server listens with mutual TLS on port 3371 of the `blockstor-apiserver` Service in `blockstor-system`. The installation created a client certificate for in-cluster clients: the Secret `blockstor-apiserver-client-tls` carries `tls.crt`, `tls.key` and `ca.crt`. A pod in `blockstor-system` can mount that Secret directly; for a driver in another namespace, see [Client certificates in another namespace](#client-certificates-in-another-namespace) first. Mount the Secret into both the CSI controller and the CSI node plugin, and set the endpoint and certificate paths: + +```yaml +env: + - name: LS_CONTROLLERS + value: "https://blockstor-apiserver.blockstor-system.svc:3371" + - name: LS_USER_CERTIFICATE + value: /etc/linstor/client/tls.crt + - name: LS_USER_KEY + value: /etc/linstor/client/tls.key + - name: LS_ROOT_CA + value: /etc/linstor/client/ca.crt +volumeMounts: + - name: client-tls + mountPath: /etc/linstor/client + readOnly: true +volumes: + - name: client-tls + secret: + secretName: blockstor-apiserver-client-tls +``` + +The node plugin's `--node` argument must match the name of a blockstor `Node`, which is the Kubernetes node name. + +## Client certificates in another namespace + +The supported layout runs the CSI driver in `blockstor-system`, next to blockstor: install piraeus-operator into that namespace, and its components take their client certificates from `blockstor-api-ca` directly, with nothing to copy. The rest of this section is for a driver that has to run elsewhere. + +The Secret `blockstor-apiserver-client-tls` and the `blockstor-api-ca` Issuer both live in `blockstor-system`, and `blockstor-api-ca` is a namespaced `Issuer`. cert-manager looks up an `Issuer` in the Certificate's own namespace, so a Certificate in any other namespace that names `blockstor-api-ca` is never issued. A pod can only mount a Secret from its own namespace. A CSI driver that runs outside `blockstor-system` needs the client material brought into its namespace. There are two ways to do that. + +**Mirror the CA into the client's namespace.** Copy the CA Secret `blockstor-api-ca` into the client's namespace and create a CA `Issuer` over it there. Certificates in that namespace are then issued and renewed by cert-manager like any other. This is what blockstor's own test installation does for piraeus-operator, in the namespace `piraeus-datastore`: + +```sh +kubectl -n blockstor-system get secret blockstor-api-ca -o json \ + | jq 'del(.metadata.namespace, .metadata.resourceVersion, .metadata.uid, .metadata.creationTimestamp, .metadata.ownerReferences, .metadata.annotations, .metadata.labels)' \ + | kubectl -n piraeus-datastore apply -f - +kubectl apply -f - <<'EOF' +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: blockstor-api-ca + namespace: piraeus-datastore +spec: + ca: + secretName: blockstor-api-ca +EOF +``` + +The copy carries the CA's private key. Anyone who can read Secrets in that namespace can sign a client certificate, and a client certificate is full access to the API. Mirror it only into a namespace you trust as much as `blockstor-system`. + +**Copy a client certificate.** Issue the client Certificate in `blockstor-system`, from the `blockstor-api-ca` Issuer, and copy the Secret it produces, with `tls.crt`, `tls.key` and `ca.crt`, into the client's namespace. The CA key stays in `blockstor-system`. The copy is not renewed: the installation's client certificate lasts 90 days and is renewed 15 days before it expires, and you have to copy the Secret again after each renewal, or the client fails once the old certificate expires. + +## With piraeus-operator + +If you install `linstor-csi` with the piraeus operator, set `LinstorCluster.spec.externalController.url` to `https://blockstor-apiserver.blockstor-system.svc:3371`. That turns off the operator's own LINSTOR controller and points the CSI driver at blockstor. + +For the client certificate, point `spec.apiTLS.certManager` at the `blockstor-api-ca` Issuer when the operator runs in `blockstor-system`. In any other namespace, mirror the CA into it as shown above and point `spec.apiTLS.certManager` at the `Issuer` you created there. The operator then creates, in its own namespace and from that Issuer, the client Certificates for itself, for the CSI pods and for its other components: + +```yaml +apiVersion: piraeus.io/v1 +kind: LinstorCluster +metadata: + name: linstorcluster +spec: + externalController: + url: https://blockstor-apiserver.blockstor-system.svc:3371 + apiTLS: + certManager: + name: blockstor-api-ca + kind: Issuer +``` + +To use copied Secrets instead, leave out `certManager` and provide a client Secret for every component the operator runs, not only for the CSI driver. The operator itself connects to the API server to configure the cluster, and the LinstorCluster does not become available without its Secret: + +| Field in `spec.apiTLS` | Default name | Used by | +|---|---|---| +| `clientSecretName` | `linstor-client-tls` | The operator itself | +| `csiControllerSecretName` | `linstor-csi-controller-tls` | The CSI controller | +| `csiNodeSecretName` | `linstor-csi-node-tls` | The CSI node plugin | +| `affinityControllerSecretName` | `linstor-affinity-controller-tls` | The affinity controller | +| `nfsServerSecretName` | `linstor-csi-nfs-server-tls` | The NFS server, if you enable it | + +Copy a client certificate into the operator's namespace under each default name, or set the fields to the names you used. Each Secret must carry `tls.crt`, `tls.key` and `ca.crt`, and you copy them again after each renewal. The mirrored CA with `certManager` avoids all of this, which is why blockstor's own test installation uses it. + +cert-manager renews the certificates it issues in place, and the API server picks up a new certificate without a restart. + +## StorageClass + +Use the `linstor.csi.linbit.com` provisioner: + +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: blockstor-replicated +provisioner: linstor.csi.linbit.com +parameters: + linstor.csi.linbit.com/storagePool: "data" + linstor.csi.linbit.com/placementCount: "2" + # Optional: take the policy from a resource group you created. + # linstor.csi.linbit.com/resourceGroup: "replicated" +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +``` + +- `storagePool` is the pool that holds the replicas with data. +- `placementCount` is the number of replicas with data. Blockstor adds a tiebreaker when that number is even. +- `resourceGroup` uses a [resource group](../resources/#resourcegroup) as the policy. Without it, `linstor-csi` does not use `DfltRscGrp`: it creates a resource group named `sc-` from the StorageClass parameters and puts the volumes there. + +`storagePool` and `placementCount` are covered by blockstor's own tests. `resourceGroup` and the other `linstor.csi.linbit.com/*` parameters are passed through to the API unchanged; test the ones you rely on in your cluster first. + +## A first PVC + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: blockstor-test +spec: + accessModes: [ReadWriteOnce] + storageClassName: blockstor-replicated + resources: + requests: + storage: 1Gi +``` + +With `WaitForFirstConsumer`, the PVC binds once a pod that uses it is scheduled. The PersistentVolume's name is also the name of the blockstor resource definition: + +```sh +kubectl get pvc blockstor-test +kubectl get pv +blockstor resource list +``` + +Delete volumes through Kubernetes: deleting the PVC makes the CSI driver delete the definition and its replicas. diff --git a/site/content/en/docs/installation.md b/site/content/en/docs/installation.md new file mode 100644 index 00000000..b3dfed8d --- /dev/null +++ b/site/content/en/docs/installation.md @@ -0,0 +1,143 @@ +--- +title: "Installation" +linkTitle: "Installation" +weight: 20 +description: "Prepare the nodes, deploy blockstor, and check that it is up." +--- + +Blockstor installs onto an existing Kubernetes cluster. You need the storage nodes prepared, cert-manager in the cluster, and the blockstor manifests from the release you want to run. + +## Prerequisites + +On every storage node that will run the satellite: + +- The DRBD 9 kernel module, loaded (`modprobe drbd`). +- For ZFS pools, the ZFS kernel module. On Talos Linux both modules come from the `siderolabs/drbd` and `siderolabs/zfs` system extensions. +- At least one spare block device, or an existing LVM volume group or ZFS pool to back the storage pools. + +The satellite image already carries the userspace tools (`drbd-utils`, `lvm2`, `cryptsetup`, `zfsutils`); only the kernel modules come from the host. + +In the cluster: + +- [cert-manager](https://cert-manager.io/), which issues the certificates for the API's mutual TLS. +- Permission to run privileged pods in the `blockstor-system` namespace. The satellite needs host networking and access to the host's block devices. + +## Get the manifests + +The CRDs and workload manifests are in the blockstor repository. Check out the tag of the release you are installing: + +```sh +git clone https://github.com/cozystack/blockstor.git +cd blockstor +git checkout v +``` + +The workload manifests carry an `__REGISTRY__/:dev` placeholder used by the development stand. Point them at the published images for your version: + +```sh +TAG= +mkdir -p rendered +for f in blockstor-deploy.yaml blockstor-apiserver-tls.yaml blockstor-apiserver-deploy.yaml blockstor-satellite-daemonset.yaml; do + sed -e "s#__REGISTRY__/\(blockstor-[a-z]*\):dev#ghcr.io/cozystack/\1:${TAG}#" "stand/$f" > "rendered/$f" +done +``` + +Each release publishes the images as `MAJOR.MINOR.PATCH` and `MAJOR.MINOR`, and stable releases also as `latest`. + +## Deploy + +Apply the pieces in this order. + +1. The custom resource definitions: + + ```sh + kubectl apply -f config/crd/bases/ + ``` + +2. The namespace, the controller and its RBAC: + + ```sh + kubectl apply -f rendered/blockstor-deploy.yaml + ``` + +3. The certificate chain for the API. It creates a CA, the API server's serving certificate, and a client certificate for in-cluster clients such as CSI: + + ```sh + kubectl apply -f rendered/blockstor-apiserver-tls.yaml + ``` + +4. The API server and the satellites: + + ```sh + kubectl apply -f rendered/blockstor-apiserver-deploy.yaml + kubectl apply -f rendered/blockstor-satellite-daemonset.yaml + ``` + +The satellite DaemonSet runs on every node that is not a control-plane node. Wait for everything to come up: + +```sh +kubectl -n blockstor-system rollout status deploy/blockstor-controller +kubectl -n blockstor-system rollout status deploy/blockstor-apiserver +kubectl -n blockstor-system rollout status daemonset/blockstor-satellite +``` + +## Register nodes and storage pools + +Each storage node is a cluster-scoped `Node` resource named after the Kubernetes node, and each pool on a node is a `StoragePool`. You can create them as YAML (see [Resources](../resources/)) or with the CLI. + +Install the CLI from the release tag. It needs Go and reads your kubeconfig, so it talks to the cluster the same way `kubectl` does. No release up to v0.1.17 has it; with those, create the objects as YAML: + +```sh +go install github.com/cozystack/blockstor/cmd/blockstor@v +``` + +Register the nodes. The address is the one DRBD uses for replication traffic between nodes: + +```sh +blockstor node create worker-1 10.0.0.11 +blockstor node create worker-2 10.0.0.12 +blockstor node create worker-3 10.0.0.13 +``` + +Create a pool from a raw disk in one step. This prepares the device and registers the pool: + +```sh +blockstor physical-storage create-device-pool zfs worker-1 /dev/sdb --pool-name data +``` + +Or register a volume group or ZFS pool that already exists: + +```sh +blockstor storage-pool create lvmthin worker-1 data my_vg/my_thinpool +``` + +The providers are `lvm`, `lvmthin`, `zfs`, `zfsthin`, `file`, `filethin` and `diskless`. Every node also gets a diskless pool, `DfltDisklessStorPool`, so it can host diskless replicas and tiebreakers. + +## Check the installation + +```sh +kubectl -n blockstor-system get pods +blockstor node list +blockstor storage-pool list +``` + +The controller, the API server replicas and one satellite per storage node should be `Running` and ready. `blockstor controller version` is no check of the installation: it prints the CLI's own version, `blockstor dev` after a plain `go install`, and does not contact the cluster. + +Every node should show as online, and every pool should report its free and total capacity once its satellite has seen it. + +Then create a first replicated volume to see the whole path work: + +```sh +blockstor resource-group create mygroup --place-count 2 --storage-pool data +blockstor volume-group create mygroup +blockstor resource-group spawn mygroup test-volume 1G +blockstor resource list +``` + +A healthy result is two `UpToDate` replicas and a `TieBreaker` on a third node. Remove it when you are done: + +```sh +blockstor resource-definition delete test-volume +``` + +Next, connect Kubernetes to it: [Provisioning volumes with CSI](../csi/). diff --git a/site/content/en/docs/layer-stack.md b/site/content/en/docs/layer-stack.md new file mode 100644 index 00000000..5d0658d9 --- /dev/null +++ b/site/content/en/docs/layer-stack.md @@ -0,0 +1,78 @@ +--- +title: "Layer stack" +linkTitle: "Layer stack" +weight: 50 +description: "Combine replication, encryption and plain local storage." +--- + +Each volume is built from layers. The bottom layer allocates space on the node, and each layer above adds something on top of the one below. The topmost layer is the device the pod uses. + +| Layer | What it does | +|---|---| +| `STORAGE` | Allocates the block device: an LVM or LVM-thin logical volume, a ZFS zvol, or a file. Every stack ends with it. | +| `LUKS` | Encrypts the device below it with `cryptsetup`. | +| `DRBD` | Replicates the device to the other nodes over the network. | + +## Supported stacks + +| Stack | Use it for | +|---|---| +| `DRBD`, `STORAGE` | Replicated volumes. This is the default. | +| `DRBD`, `LUKS`, `STORAGE` | Replicated and encrypted at rest. Each replica encrypts its own copy. | +| `LUKS`, `STORAGE` | Encrypted volume with a single replica and no DRBD. | +| `STORAGE` | Plain local volume with a single replica, for caches and scratch space. | + +These four are the only orders blockstor accepts. Other orders, such as LUKS on top of DRBD, are refused when the definition is created. + +## Choosing the stack + +A definition takes its stack from its own `layerStack`, then from its resource group's `selectFilter.layerStack`. If neither is set, it gets `DRBD`, `STORAGE`. + +Set it on a group, so every volume spawned from it gets the same stack: + +```sh +blockstor resource-group create local --place-count 1 --storage-pool data --layer-list storage +blockstor volume-group create local +blockstor resource-group spawn local scratch-1 20G +``` + +Or on one definition, in YAML: + +```yaml +apiVersion: blockstor.cozystack.io/v1alpha1 +kind: ResourceDefinition +metadata: + name: scratch-1 +spec: + layerStack: ["STORAGE"] + volumeDefinitions: + - volumeNumber: 0 + sizeKib: 20971520 +``` + +Treat the stack as fixed once a volume has replicas. Changing `layerStack` later does not encrypt or decrypt data that is already there. + +## Encryption + +LUKS volumes are opened with the cluster passphrase, which lives in a Secret in `blockstor-system`. Create it once, before the first encrypted volume: + +```sh +blockstor encryption create-passphrase '' +``` + +Keep a copy of the passphrase somewhere safe. Without it, encrypted volumes cannot be opened. Running the command again with the same passphrase is a no-op, and blockstor refuses to replace it with a different one, because that would lock every existing encrypted volume. A definition with `LUKS` in its stack and no passphrase available is refused rather than created unencrypted. + +Then create volumes with LUKS in the stack: + +```sh +blockstor resource-group create secure --place-count 2 --storage-pool data --layer-list drbd,luks,storage +blockstor volume-group create secure +blockstor resource-group spawn secure secret-volume 10G +``` + +Growing an encrypted volume resizes every layer in order: the storage below, then LUKS, then DRBD. + +Limits today: + +- All volumes of a definition share one key. +- Changing the passphrase of existing volumes is not supported. To re-key a volume, create a new one and move the data. diff --git a/site/content/en/docs/migration.md b/site/content/en/docs/migration.md new file mode 100644 index 00000000..c1ef53c8 --- /dev/null +++ b/site/content/en/docs/migration.md @@ -0,0 +1,103 @@ +--- +title: "Migrating from LINSTOR" +linkTitle: "Migrating from LINSTOR" +weight: 70 +description: "Take over an existing LINSTOR cluster in place, without copying data." +--- + +Blockstor can take over a LINSTOR cluster that runs on Kubernetes. The `linstor-migrate` tool converts LINSTOR's database into blockstor resources, and blockstor then adopts the volumes and DRBD metadata that are already on disk. No data is copied or resynced, and replica placement does not change. + +Do this in a maintenance window, after a full rehearsal on a staging cluster. + +## What moves and what does not + +Converted: + +- nodes, storage pools, resource groups, resource definitions with their volumes, replicas, and successful snapshots; +- controller-wide DRBD options; +- DRBD minor numbers, node IDs and the shared secret, unchanged; +- DRBD ports, if you capture them from the running nodes (step 3 below). + +Not converted, and listed in the tool's report: + +- **LUKS passphrases.** Encrypted volumes convert with their layers intact, but blockstor cannot open them until you set up the passphrase by hand. +- Backup shipping remotes, and replicas or snapshots that LINSTOR marked as deleted or failed. +- Snapshots of definitions with more than one volume are adopted with their first volume only. + +## Before you start + +1. **Check that the cluster is healthy.** `linstor resource list --faulty` is empty, every replica with data is `UpToDate`, and no node is offline. A degraded replica is adopted as it is and stays degraded. + +2. **Dump the LINSTOR database.** LINSTOR on Kubernetes keeps it in custom resources: + + ```sh + mkdir dump && cd dump + kubectl get crds | grep -o ".*.internal.linstor.linbit.com" \ + | xargs -I{} sh -c 'kubectl get {} -ojson > {}.json' + cd .. + ``` + +3. **Capture the DRBD ports.** LINSTOR 1.33 does not store them in its database. Without them, blockstor would assign new ports, and every replica would reconnect under load. Collect them from the satellites: + + ```sh + for pod in $(kubectl -n cozy-linstor get pods -l app.kubernetes.io/component=linstor-satellite \ + -o jsonpath='{.items[*].metadata.name}'); do + kubectl -n cozy-linstor exec "$pod" -- sh -c ' + for f in /var/lib/linstor.d/*.res; do + rd=$(basename "$f" .res) + port=$(grep -oE "address[^;]*:[0-9]+" "$f" | grep -oE "[0-9]+$" | head -1) + [ -n "$port" ] && echo "$rd $port" + done' + done | sort -u > drbd-ports.txt + ``` + + Change the namespace and label to match your installation. Every replica of a volume shares one port. If one volume shows two different ports, stop and find out why. + +4. **Take pool snapshots.** A recursive `zfs snapshot` of every pool gives you a rollback point that does not depend on either control plane. + +5. **Check the volume names on disk.** Blockstor finds existing volumes by name: `/_`, for example `data/pvc-1234_00000`. If the name does not match exactly, blockstor creates a new empty volume next to the real one. Replicated volumes would heal from their peers, but a volume with a single replica and no DRBD would come up empty. On every node, compare the list of volumes with the names the converted manifests expect: + + ```sh + zfs list -H -o name -t volume | sort + zfs list -H -o name -t snapshot | sort + ``` + + Snapshots are found as `/_00000@`. Do not continue while any single-replica volume is missing under its expected name. + +6. **Rehearse on staging.** Run the whole procedure on a copy of the pools. Check that every pool registers, every replica reaches `UpToDate` without a resync, and no new empty volumes appear. + +## Convert + +`cmd/linstor-migrate` is in no release up to v0.1.17. It ships in the release after that, so pick that one or a newer one for ``. Build the tool from the blockstor release you are installing and run it on the dump: + +```sh +go install github.com/cozystack/blockstor/cmd/linstor-migrate@v +linstor-migrate -in ./dump -drbd-ports ./drbd-ports.txt -out ./blockstor-manifests.yaml +``` + +Read the whole report it prints. Each `warning:` line is something the tool refused to guess. Resolve every `LUKS passphrase NOT migrated` and every `no DRBD port` line before you go on. + +## Cut over + +1. **Stop new volumes.** Scale the CSI provisioner and attacher to zero. +2. **Stop the LINSTOR controller.** Scale `deploy/linstor-controller` to zero. Leave the LINSTOR satellites and the DRBD devices running: they keep serving I/O. +3. **Install blockstor** as described in [Installation](../installation/), with satellites on the same nodes and pools. Check the image versions before you start them. +4. **Apply the converted manifests.** They are already in dependency order: + + ```sh + kubectl apply -f ./blockstor-manifests.yaml + ``` + +5. **Wait for adoption.** Every replica should report `UpToDate` without starting a resync, `blockstor resource list --faulty` should be empty, and every pool should report its real free space. +6. **Point CSI at blockstor**, as in [Provisioning volumes with CSI](../csi/), and scale the provisioner and attacher back up. Create a test PVC, and check that an existing PVC still mounts read-write. + +## Check before you call it done + +- The number of nodes, pools, groups, definitions and replicas matches the counts in the converter's report. +- For at least five volumes, the size, `/dev/drbd`, node IDs and port match the values from before the migration. A changed minor or node ID is a reason to stop and roll back. +- No replica is unexpectedly syncing. +- Encrypted volumes stay closed until you set up their passphrase. Keep their consumers stopped until then. + +## Roll back + +Until CSI has been pointed at blockstor and checked, rolling back means: scale blockstor to zero, scale `linstor-controller` back up, and point CSI at LINSTOR again. Blockstor did not touch the data on disk or the running DRBD devices, so this undoes only the control plane. The pool snapshots from step 4 are the last-resort rollback for the data itself. diff --git a/site/content/en/docs/networking.md b/site/content/en/docs/networking.md new file mode 100644 index 00000000..e4b1626e --- /dev/null +++ b/site/content/en/docs/networking.md @@ -0,0 +1,120 @@ +--- +title: "Networking and ports" +linkTitle: "Networking and ports" +weight: 35 +description: "Every listener blockstor opens, who connects to it, and what a firewall must allow." +--- + +Blockstor's components do not talk to each other directly. They read and write the custom resources through the Kubernetes API, and the Kubernetes API is the only thing every component needs to reach. The listeners below exist for clients, for kubelet probes, and for the data path between storage nodes. + +## Listeners + +| Component | Port | Protocol | Who connects | Reachable from | +|---|---|---|---|---| +| blockstor-apiserver | 3371 | HTTPS with mutual TLS | `linstor-csi`, piraeus-operator, `ha-controller`, the `linstor` client | The Services `blockstor-apiserver` and `blockstor-controller` in `blockstor-system` | +| blockstor-apiserver | 3370 | Plain HTTP, no authentication | `kubectl port-forward` for debugging | Every pod that can reach the API server's pod IP, unless a NetworkPolicy blocks it. Not in any Service. | +| blockstor-apiserver | 8081 | HTTP `/healthz`, `/readyz` | kubelet | The pod | +| blockstor-controller | 8081 | HTTP `/healthz`, `/readyz` | kubelet | The pod | +| blockstor-satellite | 3372 | Plain HTTP, no authentication | Other satellites, to fetch a snapshot stream | The node's own address | +| blockstor-satellite | 9101 | HTTP `/healthz`, `/readyz` | kubelet | The node's own address | +| DRBD (kernel) | 20000 to 20999 | DRBD replication, TCP | The other replicas of the same volume | The node's own address | + +The satellite runs with host networking, so its ports are open on the node itself, not on a pod IP. The manifests set `hostPort` 3372 and 9101 to make this explicit. Both have to be free on every storage node. Port 3372 is outside the DRBD port range and clear of the LINSTOR ports and of port 9100, where node-exporter usually listens on the host. + +The satellite opens no metrics listener. Its code turns off controller-runtime's metrics server, which would otherwise listen on port 8080 of the node. + +The controller has no REST listener. The manifests start it with `--enable-rest-api=false`, and the REST API is served by the separate `blockstor-apiserver` Deployment. Both the controller and the API server are started with `--metrics-bind-address=0`, so they open no metrics port. + +Satellites do not dial the controller, and the controller does not dial satellites. Both go through the Kubernetes API. + +## The API server + +Port 3371 requires a client certificate and verifies it against the CA bundle given by `--tls-client-ca-file`. The manifests create that CA with cert-manager as the `blockstor-api-ca` Issuer, and issue one client certificate into the Secret `blockstor-apiserver-client-tls`. The serving certificate covers both Service names, `blockstor-apiserver` and `blockstor-controller`, with and without the `.blockstor-system.svc.cluster.local` suffix. The API server reloads certificates on rotation without a restart. + +Port 3370 serves the same API without TLS and without authentication. It exists for debugging. The manifests start the API server with `--rest-bind-address=:3370`, which listens on every address of the pod, and they ship no NetworkPolicy. Leaving the port out of the Service only means it has no stable name: any pod that can reach an API server pod's IP gets full read and write access to the API on that port, without a certificate. + +Fence it with a NetworkPolicy. This one admits only the TLS port and the probe port to the API server pods, so port 3370 is refused from other pods: + +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: blockstor-apiserver + namespace: blockstor-system +spec: + podSelector: + matchLabels: + app: blockstor-apiserver + policyTypes: [Ingress] + ingress: + - ports: + - {protocol: TCP, port: 3371} + - {protocol: TCP, port: 8081} +``` + +A NetworkPolicy only takes effect if your CNI enforces it. `kubectl port-forward` still works with this policy in place, because it connects from inside the pod's network namespace. That is the way to reach port 3370 when you need it, for example for the `linstor` client. See [Compatibility with LINSTOR](../compatibility/#using-the-linstor-client). + +## DRBD replication + +Each volume gets one TCP port, and all of its replicas use that port to connect to each other. The controller allocates the port from a range, per node, and the port appears in the `Port` column of `kubectl get resources` and of `blockstor resource list`. + +The range is resolved in this order for every node that hosts a replica: + +1. `spec.drbdPortRange` on the blockstor `Node`, with `min` and `max`. +2. The node property `DrbdOptions/TcpPortRange`, written as `min-max`, kept for LINSTOR compatibility. +3. `TcpPortAutoRange` in `spec.extraProps` of the `ControllerConfig`. +4. The built-in default, 20000 to 20999. + +Minor numbers for the `/dev/drbd` devices follow the same pattern with `spec.drbdMinorRange` and `DrbdOptions/MinorNrRange`, with a default of 20000 to 65535. These defaults are above what LINSTOR uses, so blockstor can share nodes with a LINSTOR installation. + +A firewall between storage nodes must allow the whole port range you use, in both directions. A replica whose peer is blocked stays in `Connecting`. + +## Which address replication uses + +Each blockstor `Node` lists its addresses in `spec.netInterfaces`. `blockstor node create
` writes one interface named `default`. DRBD peers connect to that address when nothing earlier in the order below applies, so it has to be the node's own IP as seen by the other nodes. A pod IP does not work, because it is only routable inside the CNI. + +Each satellite renders the DRBD configuration of its own node, peer addresses included. Before it renders, it adds the Kubernetes node's `InternalIP` to every node's interface list as an interface named `k8s-internal`. That interface exists only in the satellite's memory and never shows up on the `Node` object. It needs the satellite's permission to read Kubernetes `nodes`, which the manifests grant. + +The address of a peer is picked in this order: + +1. `spec.satelliteEndpoint` on the blockstor `Node`, if set. +2. The interface named `k8s-internal`. +3. The interface named `default`. +4. The first interface with an address. + +This order is what keeps replication working when another tool, such as piraeus-operator, overwrites the first interface with a pod IP. + +## Snapshot stream between satellites + +When a clone or a snapshot restore places a replica on a node that does not hold the snapshot, the receiving satellite fetches the raw snapshot stream from a satellite that does, with an HTTP `GET` on port 3372. The stream has no authentication and no encryption. Keep that port reachable only between storage nodes, with a NetworkPolicy or a host firewall. + +The receiving satellite dials the address of the peer's interface named `default`, or the first interface with an address if there is no `default`. It does not use `spec.satelliteEndpoint` or the `k8s-internal` interface, so the `default` address must be the node's own IP and reachable on port 3372 from the other storage nodes. A pod IP written there by another tool breaks the stream even when DRBD still connects. + +The satellite tries every node that holds the snapshot. A peer that answers that it has no such snapshot is skipped, and if no peer has it, the replica is created empty and DRBD fills it with a full sync. A peer that cannot be reached is different: if no other peer delivers the stream, the volume is not created, the satellite fails the reconcile with a `cross-node fetch` error that names the peer's address, and retries, and the replica does not come up until the port is reachable. + +The satellite keeps running if it cannot bind port 3372, for example because another program on the node holds it. It logs `snapshot stream server exited` with the address, and other nodes can no longer fetch snapshots from it. + +## What a firewall must allow + +Between storage nodes, in both directions: + +- The DRBD port range, 20000 to 20999 by default. +- Port 3372 for the snapshot stream. + +Inside the cluster: + +- From the CSI pods and other API clients to the `blockstor-apiserver` Service on port 3371. + +On each node, for kubelet: + +- Port 9101 on the node for the satellite's probes. +- Port 8081 inside the controller and API server pods for theirs. + +Nothing needs to reach port 3370. Block it with the NetworkPolicy shown [above](#the-api-server). + +## Ports without authentication + +- 3370 on the API server. Full read and write access to the API from any pod that can reach the API server's pod IP. Not exposed by a Service, but not fenced either; add the NetworkPolicy above. +- 3372 on every storage node. Reads raw snapshot data. + +Port 3371 is the only API entry point meant for clients. diff --git a/site/content/en/docs/operations.md b/site/content/en/docs/operations.md new file mode 100644 index 00000000..ea69cf7e --- /dev/null +++ b/site/content/en/docs/operations.md @@ -0,0 +1,219 @@ +--- +title: "Day-2 operations" +linkTitle: "Operations" +weight: 65 +description: "Reading health, adding and removing nodes, growing and moving volumes, snapshots, backups and upgrades." +--- + +Everything on this page can be done with the `blockstor` CLI or by editing the custom resources with `kubectl`. The CLI commands are shown; [Resources](../resources/) has the matching YAML. + +## Reading health + +Start with the replicas that are not healthy: + +```sh +blockstor resource list --faulty +``` + +A replica counts as faulty when one of its volumes has a disk state that has not settled, or when one of its DRBD connections is anything other than `Connected`. A replica the satellite has not reported on yet is not listed, so an empty result means "nothing broken that we can see", not "everything verified". + +The custom resources carry the same information for `kubectl` and for monitoring: + +```sh +kubectl get nodes.blockstor.cozystack.io # Status ONLINE or OFFLINE +kubectl get storagepools # free and total KiB per pool +kubectl get resourcedefinitions # group, DRBD port, layers +kubectl get resources # one row per replica: node, pool, port, state, in use +``` + +Use the full name `nodes.blockstor.cozystack.io`, because plain `nodes` is the Kubernetes kind. + +A `Node` is `ONLINE` while its satellite stamps a heartbeat every 10 seconds. When the heartbeat is more than 40 seconds old, the controller sets the `Ready` condition to `Unknown` and the status to `OFFLINE`. + +The `State` column of `kubectl get resources` is the resource-level DRBD state. Per-volume disk state and per-peer connection state are in the status: + +```sh +kubectl get resource . -o jsonpath='{.status.volumes[*].diskState}' +kubectl get resource . -o jsonpath='{range .status.connections[*]}{.peerNodeName}={.message} {end}' +``` + +A `Resource` also has conditions `MetadataCreated`, `FilesystemFormatted` and `KernelLoaded`, which tell you how far a new replica has got. + +When you need the kernel's own view, run the DRBD tools inside the satellite pod of that node: + +```sh +POD=$(kubectl -n blockstor-system get pod -l app=blockstor-satellite --field-selector spec.nodeName= -o name) +kubectl -n blockstor-system exec $POD -c satellite -- drbdsetup status --verbose +``` + +Logs: + +```sh +kubectl -n blockstor-system logs -l app=blockstor-controller +kubectl -n blockstor-system logs -l app=blockstor-apiserver +kubectl -n blockstor-system logs $POD -c satellite +``` + +## Adding a node + +1. Make sure the DRBD module is loaded on the node, and the ZFS module if it will host ZFS pools. The satellite DaemonSet schedules on every node without the `node-role.kubernetes.io/control-plane` label, so the pod appears on its own. +2. Register the node. The satellite does not create the `Node` object for you. Without it, the satellite logs that the node is unregistered and waits: + + ```sh + blockstor node create worker-4 10.0.0.14 + ``` + +3. Give it a pool, from a raw disk or from an existing volume group or ZFS pool. `create-device-pool` wipes the device it is given: + + ```sh + blockstor physical-storage create-device-pool zfs worker-4 /dev/sdb --pool-name data + blockstor storage-pool create lvmthin worker-4 data my_vg/my_thinpool + ``` + +The new node shows `ONLINE` within a minute, and the pool reports its capacity once the satellite has probed it. The satellite refreshes pool capacity every 30 seconds. + +## Adding a storage pool + +A pool is one `StoragePool` per node, named `.`. Use the same pool name on every node that should take replicas of the same StorageClass, because CSI selects by pool name. The providers are `lvm`, `lvmthin`, `zfs`, `zfsthin`, `file`, `filethin` and `diskless`. Snapshots need a provider with copy-on-write: ZFS, LVM thin and thin file pools support them, thick LVM does not. `blockstor storage-pool list` shows this in the `CanSnapshots` column. + +## Taking a node out of service + +To drain a node for maintenance: + +```sh +blockstor node evacuate worker-2 +``` + +This sets the `EVICTED` flag on the `Node`. The placer stops choosing that node, and for every replica with data on it the controller first places a replacement on another node and only deletes the source once the replacement is `UpToDate`. The command refuses while a replica on the node is in use, that is, mounted and `Primary`. Stop the consumers first, or pass `--force` if you know what you are doing. + +If the node comes back before the migration finishes: + +```sh +blockstor node restore worker-2 +``` + +This only clears the flag. Replicas that have already moved are not moved back. + +Blockstor also evacuates nodes on its own. A node that has been `OFFLINE` for longer than `DrbdOptions/AutoEvictAfterTime` minutes, 60 by default, gets the `EVICTED` flag automatically. Turn this off per node or for the whole cluster with `DrbdOptions/AutoEvictAllowEviction false`. As a safety against a controller-side network problem, auto-evict stops acting when more than `DrbdOptions/AutoEvictMaxDisconnectedNodes` nodes are offline at once, 2 by default: + +```sh +blockstor controller set-property DrbdOptions/AutoEvictAfterTime 120 +blockstor node set-property worker-2 DrbdOptions/AutoEvictAllowEviction false +``` + +When a node is gone for good: + +```sh +blockstor node lost worker-2 +``` + +This deletes the node's replicas and pools, then the `Node`. It is refused while the satellite still reports `ONLINE`, unless you pass `--force`. Surviving replicas are left alone, and the controller places replacements for them. A satellite pod that restarts on that host afterwards does not re-create the `Node`; register it again with `node create` if the node returns. + +`blockstor node delete` removes only the `Node` object and refuses while replicas or pools still point at it. With `--force` it deletes them too, like `node lost`. + +## Replacing a replica with a bad disk + +If a pool's backing volume group or ZFS pool disappears, the satellite marks the `StoragePool` with `poolMissing: true`, zeroes its capacity, and `blockstor storage-pool list` shows `Faulty`. The placer skips the pool. The next successful probe clears the state, so recreating the backing pool under the same name brings it back. + +A replica whose local data is damaged shows `Inconsistent` and does not finish syncing. Rebuild it instead of repairing it. Delete the replica on that node, then let the placer put a fresh one somewhere: + +```sh +blockstor resource delete worker-3 my-volume +blockstor resource-definition auto-place my-volume --place-count 3 +``` + +`resource delete` refuses to remove the last complete copy while another replica is still syncing from it, unless you pass `--force`. The new replica syncs from the remaining `UpToDate` peers while the volume stays in use. + +## Growing a volume + +For a PVC, change `spec.resources.requests.storage`. The StorageClass must have `allowVolumeExpansion: true`. The CSI driver grows the volume definition, blockstor grows the backing devices and DRBD online, and Kubernetes grows the filesystem. + +For a volume you manage directly: + +```sh +blockstor volume-definition set-size my-volume 0 20G +``` + +Shrinking is refused. `--force` lowers the size in the definition only: the satellite grows devices and never shrinks them, so the backing devices and DRBD keep their size. + +## Moving a replica + +Move a replica with data to another node without lowering redundancy: + +```sh +blockstor resource toggle-disk worker-3 my-volume --migrate-from worker-1 +``` + +The destination gets a new replica with data. When all of its volumes are `UpToDate`, the controller deletes the source. The command refuses when the source is diskless, when the source is `Primary` and in use, or when source and destination are the same node. + +To change a single replica's kind in place: + +```sh +blockstor resource toggle-disk worker-3 my-volume --storage-pool data # give it data +blockstor resource toggle-disk worker-3 my-volume --diskless # make it diskless +``` + +`--diskless` does not check what is left: it demotes the replica even when it is the last one with data. Before you run it, make sure another replica of the volume is `UpToDate`. A conversion that is stuck can be unwound with `--cancel`. + +## Snapshots as an in-cluster backup + +Snapshots live in the same pool as the volume, so they protect against mistakes, not against losing the node or the disk. Take them before upgrades and risky changes: + +```sh +blockstor snapshot create my-volume before-upgrade +blockstor snapshot list +``` + +Restore into a new volume. The original is not touched. The new volume gets its replicas on the nodes that hold the snapshot, in the same pool, not through the autoplacer: + +```sh +blockstor snapshot resource restore \ + --from-resource my-volume --from-snapshot before-upgrade --to-resource my-volume-restored +``` + +In-place rollback is not available on purpose. A definition cannot be deleted while it still has snapshots, so delete them first: + +```sh +blockstor snapshot delete my-volume before-upgrade +``` + +## Backing up the control plane + +Blockstor keeps all of its state in custom resources, so a backup is a dump of them. Include the `ResourceDefinition` and `Resource` objects. The controller allocates DRBD minors into the definitions, in `spec.volumeDefinitions[].drbdMinor`, and DRBD ports and node IDs into the spec of each `Resource`. A restore that invents new ones would make every replica reconnect. + +```sh +for kind in controllerconfigs nodes storagepools physicaldevices \ + resourcegroups resourcedefinitions resources snapshots; do + kubectl get "$kind.blockstor.cozystack.io" -o yaml > "blockstor-$kind.yaml" +done +kubectl -n blockstor-system get secret blockstor-cluster-passphrase -o yaml > blockstor-passphrase.yaml +``` + +The full names, such as `resources.blockstor.cozystack.io`, keep `kubectl` from picking another product's resource with the same plural. The last line applies only if you use encrypted volumes. The Secret holds the cluster passphrase, and the volumes cannot be opened without it. Its name can be changed through `spec.passphraseSecretRef` on the `ControllerConfig`. Store the backup like any other secret material. + +The passphrase can also be set as a property, `DrbdOptions/EncryptPassphrase` on the controller or `DrbdOptions/Encryption/passphrase` on a definition, and a property set there wins over the Secret. A property is stored in plaintext on its resource, so the YAML dump above then carries the key to every encrypted volume. Keep the passphrase in the Secret only, and treat the dump as secret material if such a property was ever set. + +## Upgrading + +Upgrade in the same order as the installation. Update the manifests to the new tag with the same `sed` as in [Installation](../installation/#get-the-manifests), then apply. Coming from v0.1.17 or earlier, read [the note below](#upgrading-from-v0117-or-earlier) first. + +1. The CRDs, `kubectl apply -f config/crd/bases/`. New fields arrive before the code that writes them. +2. The controller, `rendered/blockstor-deploy.yaml`. +3. The API server certificates, `rendered/blockstor-apiserver-tls.yaml`. +4. The API server, `rendered/blockstor-apiserver-deploy.yaml`. It runs three replicas, so clients keep working through the rollout. +5. The satellites, `rendered/blockstor-satellite-daemonset.yaml`. + +Rolling the satellites does not interrupt I/O. The DRBD state lives in the kernel and survives the pod, the rendered DRBD configuration lives on the host under `/var/lib/blockstor-drbd.d`, and the pod is given 10 seconds to stop and does not try to disconnect DRBD on the way out. When the new satellite starts, it looks at every DRBD resource the kernel holds and tears down only the ones with no usable local disk and no live peer connection. Everything else is left for the reconciler to adjust. A replica in the middle of a sync keeps syncing. + +Watch the rollout and the result: + +```sh +kubectl -n blockstor-system rollout status daemonset/blockstor-satellite +blockstor resource list --faulty +``` + +### Upgrading from v0.1.17 or earlier + +The snapshot stream between satellites moved from port 9100 to port 3372. Open port 3372 between storage nodes before step 5; see [Networking and ports](../networking/#snapshot-stream-between-satellites). While the satellites roll, old ones listen on 9100 and new ones dial 3372, and the other way round. A clone or a snapshot restore that needs a snapshot from a satellite of the other version cannot fetch it. Its replica does not fall back to a full sync: the volume is not created, and the satellite retries until both sides run the new version. Avoid clones and restores across nodes until the rollout is done. Port 9100 can be closed afterwards. + +The API server no longer lists or watches Secrets across the cluster. It reads and writes the passphrase Secret through a Role in `blockstor-system`, which step 4 creates. diff --git a/site/content/en/docs/overview.md b/site/content/en/docs/overview.md new file mode 100644 index 00000000..d8c9b43a --- /dev/null +++ b/site/content/en/docs/overview.md @@ -0,0 +1,57 @@ +--- +title: "Overview" +linkTitle: "Overview" +weight: 10 +description: "What blockstor is, what it does today, and when to use it." +--- + +Blockstor turns the local disks of your Kubernetes nodes into replicated block storage. Each volume is a logical volume (LVM) or a zvol (ZFS) on a node, and DRBD keeps copies of it on other nodes in sync, so a pod can lose its node and come back elsewhere with its data. + +Blockstor speaks the LINSTOR REST API. The clients built for LINSTOR, `linstor-csi`, `piraeus-operator`, `ha-controller` and `golinstor`, work against it unchanged. What is different is how the cluster keeps its state: everything lives in Kubernetes custom resources, and controllers converge the cluster toward it. There is no separate database to back up and no in-memory state to lose when a pod restarts. + +## When to use it + +Blockstor fits when you want replicated block volumes for Kubernetes on your own hardware, with LVM or ZFS underneath and DRBD for replication, and you want that storage to be managed the same way as the rest of the cluster: with custom resources, `kubectl` and GitOps. + +It is also a fit if you already run LINSTOR on Kubernetes. Blockstor can take over an existing LINSTOR cluster in place, without copying data. See [Migrating from LINSTOR](../migration/). + +## What works today + +- Replicated volumes over DRBD on LVM, LVM-thin, ZFS, ZFS-thin or file backends. +- Volumes without DRBD: plain local storage, single replica. +- LUKS encryption at rest. +- Automatic placement with constraints: zones, node properties, replicas on different nodes. +- Tiebreakers and quorum policies. +- Snapshots: create, restore as a new volume, and clone. +- Snapshot shipping between nodes of the same cluster, for clones and new replicas. +- Online volume resize. +- Creating storage pools from raw disks. +- The LINSTOR-compatible REST API over mutual TLS. + +## What is not implemented + +The API answers these with `501 Not Implemented`: + +- Snapshot shipping to another cluster (disaster recovery). +- Backups: create, restore, ship, abort, and the backup queue. +- Schedules (cron-driven backups). +- Remote backends such as S3 and LINSTOR remotes. +- Other storage providers: SPDK, NVMe-oF, OpenFlex, Exos. + +## On the roadmap + +- Bring-your-own-key encryption, with Secret references in the spec. +- Shared-LUN provisioning: thick LVM plus thin qcow2 on LVM. +- A VDUSE backend through `qemu-storage-daemon`, for shared-SAN Kubernetes. + +## Components + +Blockstor runs as three images, published to GHCR on every release: + +| Image | What it runs | +|---|---| +| `ghcr.io/cozystack/blockstor-controller` | The controllers that place replicas and allocate DRBD ports, minors and node IDs. | +| `ghcr.io/cozystack/blockstor-apiserver` | The LINSTOR-compatible REST API that CSI and other clients talk to. | +| `ghcr.io/cozystack/blockstor-satellite` | The per-node agent that creates volumes and runs DRBD. | + +[Architecture](../architecture/) shows how they fit together. diff --git a/site/content/en/docs/quickstart.md b/site/content/en/docs/quickstart.md new file mode 100644 index 00000000..f23cb60f --- /dev/null +++ b/site/content/en/docs/quickstart.md @@ -0,0 +1,171 @@ +--- +title: "Quickstart" +linkTitle: "Quickstart" +weight: 15 +description: "From an empty cluster to a pod writing to a replicated volume." +--- + +This page is the shortest path from a Kubernetes cluster with DRBD-capable nodes to a pod writing to a replicated volume. It leaves out the choices. [Installation](../installation/) and [Provisioning volumes with CSI](../csi/) explain them. + +## Before you start + +The `blockstor` CLI this page uses is in no release up to v0.1.17. It ships in the release after that, so pick that one or a newer one for ``. + +You need: + +- Three or more worker nodes with the DRBD 9 kernel module loaded, and the ZFS module if you want ZFS pools. On Talos Linux both come from the `siderolabs/drbd` and `siderolabs/zfs` system extensions. +- One spare block device on each of those nodes. The example below uses `/dev/sdb`. +- [cert-manager](https://cert-manager.io/) in the cluster. It issues the certificates for the API. +- `kubectl` with cluster-admin access, and Go to install the `blockstor` CLI. + +The satellite image carries `drbd-utils`, `lvm2`, `cryptsetup` and `zfsutils`. Only the kernel modules come from the host. + +## 1. Deploy blockstor + +Check out the release tag and point the manifests at the published images: + +```sh +git clone https://github.com/cozystack/blockstor.git +cd blockstor +git checkout v + +TAG= +mkdir -p rendered +for f in blockstor-deploy.yaml blockstor-apiserver-tls.yaml blockstor-apiserver-deploy.yaml blockstor-satellite-daemonset.yaml; do + sed -e "s#__REGISTRY__/\(blockstor-[a-z]*\):dev#ghcr.io/cozystack/\1:${TAG}#" "stand/$f" > "rendered/$f" +done +``` + +Apply the pieces in this order: + +```sh +kubectl apply -f config/crd/bases/ +kubectl apply -f rendered/blockstor-deploy.yaml +kubectl apply -f rendered/blockstor-apiserver-tls.yaml +kubectl apply -f rendered/blockstor-apiserver-deploy.yaml +kubectl apply -f rendered/blockstor-satellite-daemonset.yaml +``` + +Wait until everything is running. The satellite DaemonSet lands on every node that is not a control-plane node: + +```sh +kubectl -n blockstor-system rollout status deploy/blockstor-controller +kubectl -n blockstor-system rollout status deploy/blockstor-apiserver +kubectl -n blockstor-system rollout status daemonset/blockstor-satellite +``` + +## 2. Register nodes and pools + +Install the CLI. It reads your kubeconfig and needs no port-forward: + +```sh +go install github.com/cozystack/blockstor/cmd/blockstor@v +``` + +Register each storage node under its Kubernetes node name. The address is the node's own IP, the one other nodes can reach, and it is what DRBD replicates over. `kubectl get nodes -o wide` shows it as `INTERNAL-IP`: + +```sh +blockstor node create worker-1 10.0.0.11 +blockstor node create worker-2 10.0.0.12 +blockstor node create worker-3 10.0.0.13 +``` + +Turn the spare disk on each node into a pool called `data`. This wipes the disk, creates the ZFS pool and registers it in one step: + +```sh +blockstor physical-storage create-device-pool zfs worker-1 /dev/sdb --pool-name data +blockstor physical-storage create-device-pool zfs worker-2 /dev/sdb --pool-name data +blockstor physical-storage create-device-pool zfs worker-3 /dev/sdb --pool-name data +``` + +Check the result. Every node should be `ONLINE` and every pool should show free and total capacity: + +```sh +blockstor node list +blockstor storage-pool list +``` + +## 3. Connect the CSI driver + +Kubernetes provisions blockstor volumes through `linstor-csi`. Deploy the driver's controller and node plugin, and point them at the blockstor API server with the client certificate the installation created: + +- Endpoint: `https://blockstor-apiserver.blockstor-system.svc:3371` +- Client certificate: the Secret `blockstor-apiserver-client-tls` in `blockstor-system`, with `tls.crt`, `tls.key` and `ca.crt` + +The exact environment variables and volume mounts, and the piraeus-operator alternative, are in [Provisioning volumes with CSI](../csi/#point-the-driver-at-blockstor). + +Then create a StorageClass that places two replicas with data in the `data` pool: + +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: blockstor-replicated +provisioner: linstor.csi.linbit.com +parameters: + linstor.csi.linbit.com/storagePool: "data" + linstor.csi.linbit.com/placementCount: "2" +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +``` + +Two replicas is an even number, so blockstor adds a tiebreaker on a third node. + +## 4. Create a volume and write to it + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: quickstart +spec: + accessModes: [ReadWriteOnce] + storageClassName: blockstor-replicated + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: quickstart +spec: + containers: + - name: writer + image: busybox + command: ["sh", "-c", "echo hello from blockstor > /data/hello && sleep 3600"] + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: quickstart +``` + +With `WaitForFirstConsumer`, the volume is created when the pod is scheduled. + +## 5. Check that it worked + +```sh +kubectl get pvc quickstart +kubectl get pv +blockstor resource list +kubectl exec quickstart -- cat /data/hello +``` + +What you should see: + +- The PVC is `Bound`. +- The PersistentVolume has the same name as a blockstor resource definition. +- `blockstor resource list` shows two replicas in `UpToDate` and one `TieBreaker`, on three different nodes. +- The file reads back. + +Delete the pod and the PVC when you are done. The CSI driver deletes the definition and its replicas with the PVC. + +## Where next + +- [Networking and ports](../networking/) for what a firewall between nodes must allow. +- [Using the CLI](../cli/) and [Resources](../resources/) for day-to-day work. +- [Day-2 operations](../operations/) for adding nodes, moving replicas and upgrading. +- [Troubleshooting](../troubleshooting/) when something does not converge. diff --git a/site/content/en/docs/resources.md b/site/content/en/docs/resources.md new file mode 100644 index 00000000..77b59b7f --- /dev/null +++ b/site/content/en/docs/resources.md @@ -0,0 +1,201 @@ +--- +title: "Resources" +linkTitle: "Resources" +weight: 40 +description: "The blockstor custom resources, with examples." +--- + +Everything blockstor manages is a cluster-scoped custom resource in the `blockstor.cozystack.io/v1alpha1` API group. You can create and change them with `kubectl` or GitOps, with the [`blockstor` CLI](../cli/), or through the REST API. All three edit the same objects. + +| Kind | What it describes | Name | +|---|---|---| +| `Node` | A storage node and its replication address | the Kubernetes node name | +| `StoragePool` | A pool of space on one node | `.` | +| `PhysicalDevice` | A raw disk the satellite found on a node | set by the satellite | +| `ResourceGroup` | A placement and replication policy | any | +| `ResourceDefinition` | One volume set: sizes, layers, policy | any | +| `Resource` | One replica of a definition on one node | `.` | +| `Snapshot` | A snapshot of a definition | `.` | +| `ControllerConfig` | Cluster-wide settings | always `default` | + +The naming rules are enforced by the API server, so a `Resource` or `StoragePool` with a name that does not match its spec is refused. Each kind has printer columns, so `kubectl get resources` or `kubectl get storagepools` gives a useful table. + +Most objects have a `spec` you write and a `status` that blockstor writes. The examples below show only `spec`. + +## Node + +A node that runs the satellite. DRBD replication uses one of its addresses: `spec.satelliteEndpoint` if set, then the Kubernetes node's `InternalIP` that the satellite adds as `k8s-internal`, then the interface named `default`, then the first interface with an address. See [Networking and ports](../networking/#which-address-replication-uses). + +```yaml +apiVersion: blockstor.cozystack.io/v1alpha1 +kind: Node +metadata: + name: worker-1 +spec: + type: SATELLITE + netInterfaces: + - name: default + address: 10.0.0.11 +``` + +## StoragePool + +Space for replicas on one node. The provider decides how volumes are allocated, and the properties name the backing volume group, thin pool, ZFS pool or directory. + +```yaml +apiVersion: blockstor.cozystack.io/v1alpha1 +kind: StoragePool +metadata: + name: data.worker-1 +spec: + nodeName: worker-1 + poolName: data + providerKind: LVM_THIN + props: + StorDriver/LvmVg: my_vg + StorDriver/ThinPool: my_thinpool +``` + +For a ZFS thin pool: + +```yaml +apiVersion: blockstor.cozystack.io/v1alpha1 +kind: StoragePool +metadata: + name: zfs-data.worker-1 +spec: + nodeName: worker-1 + poolName: zfs-data + providerKind: ZFS_THIN + props: + StorDriver/ZPoolThin: tank +``` + +The providers are `LVM`, `LVM_THIN`, `ZFS`, `ZFS_THIN`, `FILE`, `FILE_THIN` and `DISKLESS`. Pools with the same name on different nodes form one pool from the placement point of view: a policy that says "pool `data`" can place replicas on every node that has a `data` pool. + +## PhysicalDevice + +The satellites report the raw disks they find as `PhysicalDevice` objects, with their size, model and path in `status`. To build a pool from one, set `attachTo`: + +```yaml +apiVersion: blockstor.cozystack.io/v1alpha1 +kind: PhysicalDevice +metadata: + name: +spec: + attachTo: + storagePoolName: data + providerKind: ZFS_THIN + zPoolName: data +``` + +The CLI does the same thing with `blockstor physical-storage create-device-pool`. + +## ResourceGroup + +A policy that volumes inherit: how many replicas, in which pool, with which layers, and with which DRBD options. + +```yaml +apiVersion: blockstor.cozystack.io/v1alpha1 +kind: ResourceGroup +metadata: + name: replicated +spec: + description: Two data replicas plus a tiebreaker + selectFilter: + placeCount: 2 + storagePool: data + layerStack: ["DRBD", "STORAGE"] + replicasOnDifferent: ["Aux/topology.kubernetes.io/zone"] + drbdOptions: + resource: + quorum: majority + onNoQuorum: suspend-io + volumeGroups: + - volumeNumber: 0 +``` + +Placement constraints match node properties. Blockstor copies the Kubernetes node labels under `topology.kubernetes.io/` and `node-role.kubernetes.io/`, and `kubernetes.io/hostname`, onto each `Node` as `Aux/