feat: add support for distributed nodes

README.md

@@ -12,6 +12,8 @@ RustFS NixOS module with secure secret management and systemd hardening.
 - **[docs/MIGRATION.md](./docs/MIGRATION.md)** - Migrating from old insecure configuration
 - **[docs/IMPROVEMENTS.md](./docs/IMPROVEMENTS.md)** - Technical implementation details
 - **[examples/nixos-configuration.nix](./examples/nixos-configuration.nix)** - Example secure configuration
+- **[examples/single-node-multi-disk.nix](./examples/single-node-multi-disk.nix)** - One node, four drives (erasure coding)
+- **[examples/distributed-cluster.nix](./examples/distributed-cluster.nix)** - Four-node cluster, four drives per node
 
 ## Features
 
@@ -172,7 +174,66 @@ Group under which RustFS runs.
 
 **Default:** `["/var/lib/rustfs"]`
 
-List of paths or comma-separated string where RustFS stores data. Use persistent locations, not /tmp.
+List of paths or comma-separated string where RustFS stores data. Use persistent locations, not /tmp. Each entry must be
+its own filesystem; several entries on one disk give no redundancy. Erasure coding needs at least 4 drives. Ignored when
+`distributed.enable` is set — use `distributed.volumes` instead.
+
+### services.rustfs.distributed.enable
+
+**Type:** `bool`
+
+**Default:** `false`
+
+Whether to run as part of a distributed RustFS cluster spanning several nodes. When enabled, the module renders the
+shared endpoint list (`http://<node>:<port><volume>` for every node × volume pair) that all nodes must agree on, ordered
+drive-major so an erasure set spans nodes instead of sitting on one.
+
+See [examples/distributed-cluster.nix](./examples/distributed-cluster.nix) for a complete four-node configuration.
+
+### services.rustfs.distributed.nodes
+
+**Type:** `list of strings`
+
+**Default:** `[]`
+
+**Example:** `["node1" "node2" "node3" "node4"]`
+
+Hostnames of every node in the cluster, resolvable from each of them. Set identically on all nodes — the endpoint list is
+rendered from this and must come out byte-identical cluster-wide. At least 4 nodes are required.
+
+### services.rustfs.distributed.volumes
+
+**Type:** `list of strings`
+
+**Default:** `[]`
+
+**Example:** `["/mnt/disk0" "/mnt/disk1" "/mnt/disk2" "/mnt/disk3"]`
+
+Drive paths present on each node, each on its own filesystem. Every node uses the same layout, so this replaces
+`volumes` in distributed mode and is what gets created and made writable locally. At least 4 drives per node are
+required.
+
+### services.rustfs.distributed.port
+
+**Type:** `port`
+
+**Default:** `9000`
+
+Port peers reach each other on. Must match the port in `address`, and be open between nodes in the firewall.
+
+### services.rustfs.distributed.localEndpointHost
+
+**Type:** `string`
+
+**Default:** `config.networking.hostName`
+
+Which entry of `nodes` identifies this machine, so it claims its own drives instead of reaching them over RPC. Required
+whenever `address` binds a wildcard such as `0.0.0.0`, since RustFS cannot infer its identity from that and would
+otherwise treat every drive as remote.
+
+> **Note**: All nodes must share the *same* access/secret key pair, and it must not be the default
+> `rustfsadmin`/`rustfsadmin` — RustFS derives the inter-node RPC secret from the credentials and refuses to derive one
+> from the defaults.
 
 ### services.rustfs.address
 

examples/distributed-cluster.nix

@@ -0,0 +1,118 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# RustFS distributed cluster NixOS Configuration Example
+#
+# Four nodes with four drives each. This exact file is imported by every node —
+# `services.rustfs.distributed` is deliberately identical everywhere, and the
+# module derives each node's identity from `networking.hostName`.
+#
+# Requirements the module asserts at evaluation time:
+#   - at least 4 nodes in `distributed.nodes`
+#   - at least 4 drives in `distributed.volumes`
+#   - `distributed.localEndpointHost` is one of `distributed.nodes`
+#
+# Additional requirements the module cannot check for you:
+#   - every hostname in `nodes` resolves from every other node (DNS or
+#     `networking.hosts`), and `distributed.port` is reachable between them
+#   - the access/secret key pair is identical on all nodes, and is *not* the
+#     default rustfsadmin/rustfsadmin — RustFS derives the inter-node RPC secret
+#     from the credentials and refuses to derive one from the defaults
+#   - each drive is its own filesystem, on every node
+#
+# For complete security documentation, see ../docs/SECURITY.md
+
+{ config, lib, pkgs, ... }:
+
+let
+  # Same on every node: the shared endpoint list is rendered from these, and it
+  # must come out byte-identical cluster-wide.
+  nodes = [
+    "node1"
+    "node2"
+    "node3"
+    "node4"
+  ];
+
+  volumes = [
+    "/mnt/rustfs0"
+    "/mnt/rustfs1"
+    "/mnt/rustfs2"
+    "/mnt/rustfs3"
+  ];
+in
+{
+  # Drive layout, identical on each node. Replace the by-id device names with the
+  # ones on your hardware; if they differ per node, split this attrset out into a
+  # per-host file and keep the mount points the same.
+  fileSystems = lib.listToAttrs (
+    lib.imap0
+      (i: volume: lib.nameValuePair volume {
+        device = "/dev/disk/by-id/REPLACE-ME-disk${toString i}";
+        fsType = "xfs";
+      })
+      volumes
+  );
+
+  services.rustfs = {
+    enable = true;
+
+    distributed = {
+      enable = true;
+
+      # Replaces `volumes` when distributed mode is on: these are the local
+      # drives, and the module expands them into the shared endpoint list
+      # (http://<node>:<port><volume>) that every node must agree on.
+      inherit nodes volumes;
+
+      port = 9000;
+
+      # Which entry of `nodes` is this machine. The default is
+      # config.networking.hostName, which is correct as long as the hostname
+      # matches the name used in `nodes`. Set it explicitly when it does not —
+      # required whenever `address` binds a wildcard, since RustFS cannot infer
+      # its own identity from 0.0.0.0 and would treat every drive as remote.
+      localEndpointHost = config.networking.hostName;
+    };
+
+    # Must bind an address peers can reach, on `distributed.port`.
+    address = "0.0.0.0:9000";
+
+    # SECURITY: Bind console to localhost only, access via SSH tunnel
+    consoleEnable = true;
+    consoleAddress = "127.0.0.1:9001";
+
+    logLevel = "info";
+
+    # SECURITY: Use file-based secrets, never plain text!
+    # Distribute the *same* pair to every node, e.g. with sops-nix or agenix.
+    accessKeyFile = "/run/secrets/rustfs-access-key";
+    secretKeyFile = "/run/secrets/rustfs-secret-key";
+  };
+
+  # Peers talk to each other on the API port, so it has to be open between nodes.
+  # Narrow this to the cluster subnet in production rather than opening it wide.
+  networking.firewall = {
+    enable = true;
+    allowedTCPPorts = [ 9000 ];
+  };
+
+  # Only needed if the node names are not in DNS.
+  # networking.hosts = {
+  #   "10.0.0.1" = [ "node1" ];
+  #   "10.0.0.2" = [ "node2" ];
+  #   "10.0.0.3" = [ "node3" ];
+  #   "10.0.0.4" = [ "node4" ];
+  # };
+}

examples/single-node-multi-disk.nix

@@ -0,0 +1,81 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# RustFS single-node, multi-disk NixOS Configuration Example
+#
+# One machine with four drives, so erasure coding has something to spread parity
+# across. Erasure coding needs at least four drives; with fewer, RustFS falls back
+# to a single-drive layout with no redundancy.
+#
+# Each volume must sit on its own filesystem — pointing several `volumes` entries
+# at directories of the same disk gives you no redundancy at all, and RustFS
+# rejects the layout.
+#
+# For complete security documentation, see ../docs/SECURITY.md
+
+{ config, pkgs, ... }:
+
+{
+  # The drives themselves. Replace the /dev/disk/by-id/… devices with the ones on
+  # your machine; by-id names are stable across reboots, unlike /dev/sdX.
+  fileSystems = {
+    "/mnt/rustfs0" = {
+      device = "/dev/disk/by-id/REPLACE-ME-disk0";
+      fsType = "xfs";
+    };
+    "/mnt/rustfs1" = {
+      device = "/dev/disk/by-id/REPLACE-ME-disk1";
+      fsType = "xfs";
+    };
+    "/mnt/rustfs2" = {
+      device = "/dev/disk/by-id/REPLACE-ME-disk2";
+      fsType = "xfs";
+    };
+    "/mnt/rustfs3" = {
+      device = "/dev/disk/by-id/REPLACE-ME-disk3";
+      fsType = "xfs";
+    };
+  };
+
+  services.rustfs = {
+    enable = true;
+
+    # One entry per drive. A comma-separated string works too, but the list form
+    # is easier to read and to generate.
+    volumes = [
+      "/mnt/rustfs0"
+      "/mnt/rustfs1"
+      "/mnt/rustfs2"
+      "/mnt/rustfs3"
+    ];
+
+    address = ":9000";
+
+    # SECURITY: Bind console to localhost only, access via SSH tunnel
+    consoleEnable = true;
+    consoleAddress = "127.0.0.1:9001";
+
+    logLevel = "info";
+
+    # SECURITY: Use file-based secrets, never plain text!
+    # See nixos-configuration.nix for the sops-nix / agenix variants.
+    accessKeyFile = "/run/secrets/rustfs-access-key";
+    secretKeyFile = "/run/secrets/rustfs-secret-key";
+  };
+
+  networking.firewall = {
+    enable = true;
+    allowedTCPPorts = [ 9000 ];
+  };
+}

flake.nix

@@ -30,6 +30,9 @@
         "aarch64-darwin"
       ];
       forAllSystems = nixpkgs.lib.genAttrs supportedSystems;
+      forLinuxSystems = nixpkgs.lib.genAttrs (
+        nixpkgs.lib.filter (nixpkgs.lib.hasSuffix "-linux") supportedSystems
+      );
     in
     {
       # Standard NixOS Module
@@ -98,6 +101,13 @@
         }
       );
 
+      checks = forLinuxSystems (system:
+        import ./tests {
+          inherit self;
+          pkgs = import nixpkgs { inherit system; };
+        }
+      );
+
       devShells = forAllSystems (system:
         let pkgs = import nixpkgs { inherit system; };
         in {