From 81cb02b040895086c597557ed87c90b16ed1c548 Mon Sep 17 00:00:00 2001 From: Christoph Hollizeck Date: Thu, 6 Aug 2026 22:14:33 +0200 Subject: [PATCH] feat: add support for distributed nodes I was playing around with colmena trying to deploy buckets, and then I thought instead of just doing this stuff only for myself I should upstream it since I was already running the vm tests I thought this would be a good idea to introduce these here aswell full disclosure I let claude write the documentation parts because I was lazy, and I used it for debugging and simplifying my vm tests to match this repo better --- README.md | 63 +++++++++++- examples/distributed-cluster.nix | 118 ++++++++++++++++++++++ examples/single-node-multi-disk.nix | 81 +++++++++++++++ flake.nix | 10 ++ nixos/rustfs.nix | 150 +++++++++++++++++++++++----- tests/default.nix | 6 ++ tests/multi-node-multi-disk.nix | 112 +++++++++++++++++++++ tests/single-node-multi-disk.nix | 68 +++++++++++++ 8 files changed, 581 insertions(+), 27 deletions(-) create mode 100644 examples/distributed-cluster.nix create mode 100644 examples/single-node-multi-disk.nix create mode 100644 tests/default.nix create mode 100644 tests/multi-node-multi-disk.nix create mode 100644 tests/single-node-multi-disk.nix diff --git a/README.md b/README.md index 309cd07..77d97f8 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ RustFS NixOS module with secure secret management and systemd hardening. - **[docs/MIGRATION.md](./docs/MIGRATION.md)** - Migrating from old insecure configuration - **[docs/IMPROVEMENTS.md](./docs/IMPROVEMENTS.md)** - Technical implementation details - **[examples/nixos-configuration.nix](./examples/nixos-configuration.nix)** - Example secure configuration +- **[examples/single-node-multi-disk.nix](./examples/single-node-multi-disk.nix)** - One node, four drives (erasure coding) +- **[examples/distributed-cluster.nix](./examples/distributed-cluster.nix)** - Four-node cluster, four drives per node ## Features @@ -172,7 +174,66 @@ Group under which RustFS runs. **Default:** `["/var/lib/rustfs"]` -List of paths or comma-separated string where RustFS stores data. Use persistent locations, not /tmp. +List of paths or comma-separated string where RustFS stores data. Use persistent locations, not /tmp. Each entry must be +its own filesystem; several entries on one disk give no redundancy. Erasure coding needs at least 4 drives. Ignored when +`distributed.enable` is set — use `distributed.volumes` instead. + +### services.rustfs.distributed.enable + +**Type:** `bool` + +**Default:** `false` + +Whether to run as part of a distributed RustFS cluster spanning several nodes. When enabled, the module renders the +shared endpoint list (`http://:` for every node × volume pair) that all nodes must agree on, ordered +drive-major so an erasure set spans nodes instead of sitting on one. + +See [examples/distributed-cluster.nix](./examples/distributed-cluster.nix) for a complete four-node configuration. + +### services.rustfs.distributed.nodes + +**Type:** `list of strings` + +**Default:** `[]` + +**Example:** `["node1" "node2" "node3" "node4"]` + +Hostnames of every node in the cluster, resolvable from each of them. Set identically on all nodes — the endpoint list is +rendered from this and must come out byte-identical cluster-wide. At least 4 nodes are required. + +### services.rustfs.distributed.volumes + +**Type:** `list of strings` + +**Default:** `[]` + +**Example:** `["/mnt/disk0" "/mnt/disk1" "/mnt/disk2" "/mnt/disk3"]` + +Drive paths present on each node, each on its own filesystem. Every node uses the same layout, so this replaces +`volumes` in distributed mode and is what gets created and made writable locally. At least 4 drives per node are +required. + +### services.rustfs.distributed.port + +**Type:** `port` + +**Default:** `9000` + +Port peers reach each other on. Must match the port in `address`, and be open between nodes in the firewall. + +### services.rustfs.distributed.localEndpointHost + +**Type:** `string` + +**Default:** `config.networking.hostName` + +Which entry of `nodes` identifies this machine, so it claims its own drives instead of reaching them over RPC. Required +whenever `address` binds a wildcard such as `0.0.0.0`, since RustFS cannot infer its identity from that and would +otherwise treat every drive as remote. + +> **Note**: All nodes must share the *same* access/secret key pair, and it must not be the default +> `rustfsadmin`/`rustfsadmin` — RustFS derives the inter-node RPC secret from the credentials and refuses to derive one +> from the defaults. ### services.rustfs.address diff --git a/examples/distributed-cluster.nix b/examples/distributed-cluster.nix new file mode 100644 index 0000000..85792e3 --- /dev/null +++ b/examples/distributed-cluster.nix @@ -0,0 +1,118 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# RustFS distributed cluster NixOS Configuration Example +# +# Four nodes with four drives each. This exact file is imported by every node — +# `services.rustfs.distributed` is deliberately identical everywhere, and the +# module derives each node's identity from `networking.hostName`. +# +# Requirements the module asserts at evaluation time: +# - at least 4 nodes in `distributed.nodes` +# - at least 4 drives in `distributed.volumes` +# - `distributed.localEndpointHost` is one of `distributed.nodes` +# +# Additional requirements the module cannot check for you: +# - every hostname in `nodes` resolves from every other node (DNS or +# `networking.hosts`), and `distributed.port` is reachable between them +# - the access/secret key pair is identical on all nodes, and is *not* the +# default rustfsadmin/rustfsadmin — RustFS derives the inter-node RPC secret +# from the credentials and refuses to derive one from the defaults +# - each drive is its own filesystem, on every node +# +# For complete security documentation, see ../docs/SECURITY.md + +{ config, lib, pkgs, ... }: + +let + # Same on every node: the shared endpoint list is rendered from these, and it + # must come out byte-identical cluster-wide. + nodes = [ + "node1" + "node2" + "node3" + "node4" + ]; + + volumes = [ + "/mnt/rustfs0" + "/mnt/rustfs1" + "/mnt/rustfs2" + "/mnt/rustfs3" + ]; +in +{ + # Drive layout, identical on each node. Replace the by-id device names with the + # ones on your hardware; if they differ per node, split this attrset out into a + # per-host file and keep the mount points the same. + fileSystems = lib.listToAttrs ( + lib.imap0 + (i: volume: lib.nameValuePair volume { + device = "/dev/disk/by-id/REPLACE-ME-disk${toString i}"; + fsType = "xfs"; + }) + volumes + ); + + services.rustfs = { + enable = true; + + distributed = { + enable = true; + + # Replaces `volumes` when distributed mode is on: these are the local + # drives, and the module expands them into the shared endpoint list + # (http://:) that every node must agree on. + inherit nodes volumes; + + port = 9000; + + # Which entry of `nodes` is this machine. The default is + # config.networking.hostName, which is correct as long as the hostname + # matches the name used in `nodes`. Set it explicitly when it does not — + # required whenever `address` binds a wildcard, since RustFS cannot infer + # its own identity from 0.0.0.0 and would treat every drive as remote. + localEndpointHost = config.networking.hostName; + }; + + # Must bind an address peers can reach, on `distributed.port`. + address = "0.0.0.0:9000"; + + # SECURITY: Bind console to localhost only, access via SSH tunnel + consoleEnable = true; + consoleAddress = "127.0.0.1:9001"; + + logLevel = "info"; + + # SECURITY: Use file-based secrets, never plain text! + # Distribute the *same* pair to every node, e.g. with sops-nix or agenix. + accessKeyFile = "/run/secrets/rustfs-access-key"; + secretKeyFile = "/run/secrets/rustfs-secret-key"; + }; + + # Peers talk to each other on the API port, so it has to be open between nodes. + # Narrow this to the cluster subnet in production rather than opening it wide. + networking.firewall = { + enable = true; + allowedTCPPorts = [ 9000 ]; + }; + + # Only needed if the node names are not in DNS. + # networking.hosts = { + # "10.0.0.1" = [ "node1" ]; + # "10.0.0.2" = [ "node2" ]; + # "10.0.0.3" = [ "node3" ]; + # "10.0.0.4" = [ "node4" ]; + # }; +} diff --git a/examples/single-node-multi-disk.nix b/examples/single-node-multi-disk.nix new file mode 100644 index 0000000..a635e97 --- /dev/null +++ b/examples/single-node-multi-disk.nix @@ -0,0 +1,81 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# RustFS single-node, multi-disk NixOS Configuration Example +# +# One machine with four drives, so erasure coding has something to spread parity +# across. Erasure coding needs at least four drives; with fewer, RustFS falls back +# to a single-drive layout with no redundancy. +# +# Each volume must sit on its own filesystem — pointing several `volumes` entries +# at directories of the same disk gives you no redundancy at all, and RustFS +# rejects the layout. +# +# For complete security documentation, see ../docs/SECURITY.md + +{ config, pkgs, ... }: + +{ + # The drives themselves. Replace the /dev/disk/by-id/… devices with the ones on + # your machine; by-id names are stable across reboots, unlike /dev/sdX. + fileSystems = { + "/mnt/rustfs0" = { + device = "/dev/disk/by-id/REPLACE-ME-disk0"; + fsType = "xfs"; + }; + "/mnt/rustfs1" = { + device = "/dev/disk/by-id/REPLACE-ME-disk1"; + fsType = "xfs"; + }; + "/mnt/rustfs2" = { + device = "/dev/disk/by-id/REPLACE-ME-disk2"; + fsType = "xfs"; + }; + "/mnt/rustfs3" = { + device = "/dev/disk/by-id/REPLACE-ME-disk3"; + fsType = "xfs"; + }; + }; + + services.rustfs = { + enable = true; + + # One entry per drive. A comma-separated string works too, but the list form + # is easier to read and to generate. + volumes = [ + "/mnt/rustfs0" + "/mnt/rustfs1" + "/mnt/rustfs2" + "/mnt/rustfs3" + ]; + + address = ":9000"; + + # SECURITY: Bind console to localhost only, access via SSH tunnel + consoleEnable = true; + consoleAddress = "127.0.0.1:9001"; + + logLevel = "info"; + + # SECURITY: Use file-based secrets, never plain text! + # See nixos-configuration.nix for the sops-nix / agenix variants. + accessKeyFile = "/run/secrets/rustfs-access-key"; + secretKeyFile = "/run/secrets/rustfs-secret-key"; + }; + + networking.firewall = { + enable = true; + allowedTCPPorts = [ 9000 ]; + }; +} diff --git a/flake.nix b/flake.nix index 8562950..8ad5e9c 100644 --- a/flake.nix +++ b/flake.nix @@ -30,6 +30,9 @@ "aarch64-darwin" ]; forAllSystems = nixpkgs.lib.genAttrs supportedSystems; + forLinuxSystems = nixpkgs.lib.genAttrs ( + nixpkgs.lib.filter (nixpkgs.lib.hasSuffix "-linux") supportedSystems + ); in { # Standard NixOS Module @@ -98,6 +101,13 @@ } ); + checks = forLinuxSystems (system: + import ./tests { + inherit self; + pkgs = import nixpkgs { inherit system; }; + } + ); + devShells = forAllSystems (system: let pkgs = import nixpkgs { inherit system; }; in { diff --git a/nixos/rustfs.nix b/nixos/rustfs.nix index e056a53..45fbc06 100644 --- a/nixos/rustfs.nix +++ b/nixos/rustfs.nix @@ -21,16 +21,25 @@ let cfg = config.services.rustfs; - # Helper to handle volumes as list or string - volumesStr = - if builtins.isList cfg.volumes - then lib.concatStringsSep "," cfg.volumes - else cfg.volumes; - - volumesList = - if builtins.isList cfg.volumes - then cfg.volumes - else [ cfg.volumes ]; + dist = cfg.distributed; + + configuredVolumes = + if builtins.isList cfg.volumes then + cfg.volumes + else + lib.filter (v: v != "") (lib.splitString "," cfg.volumes); + + localVolumes = if dist.enable then dist.volumes else configuredVolumes; + + # Every node must be given the identical endpoint list, ordered drive-major so an + # erasure set spans nodes instead of sitting on one. + endpoints = lib.concatMap + ( + volume: map (node: "http://${node}:${toString dist.port}${volume}") dist.nodes + ) + dist.volumes; + + volumesStr = lib.concatStringsSep " " (if dist.enable then endpoints else configuredVolumes); in { imports = [ @@ -101,6 +110,59 @@ in description = "List of paths or comma-separated string where RustFS stores data."; }; + distributed = { + enable = lib.mkEnableOption "a distributed RustFS cluster spanning several nodes"; + + nodes = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ + "node1" + "node2" + "node3" + "node4" + ]; + description = '' + Hostnames of every node in the cluster, resolvable from each of them. + Used to render the shared endpoint list; set identically on all nodes. + ''; + }; + + volumes = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ + "/mnt/disk0" + "/mnt/disk1" + "/mnt/disk2" + "/mnt/disk3" + ]; + description = '' + Drive paths present on each node, each on its own filesystem. Every node + uses the same layout, so this replaces `volumes` when distributed mode is + enabled and is what gets created and made writable locally. + ''; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 9000; + description = "Port peers reach each other on, matching `address`."; + }; + + localEndpointHost = lib.mkOption { + type = lib.types.str; + default = config.networking.hostName; + defaultText = lib.literalExpression "config.networking.hostName"; + description = '' + Which entry of `nodes` identifies this machine, so it claims its own + drives instead of reaching them over RPC. Required whenever `address` + binds a wildcard such as `0.0.0.0`, since RustFS cannot infer its + identity from that and would otherwise treat every drive as remote. + ''; + }; + }; + address = lib.mkOption { type = lib.types.str; default = ":9000"; @@ -143,6 +205,22 @@ in }; config = lib.mkIf cfg.enable { + # Erasure coding needs 4 drives; a distributed cluster needs 4 nodes of 4. + assertions = [ + { + assertion = dist.enable -> builtins.length dist.nodes >= 4; + message = "services.rustfs.distributed.nodes needs at least 4 nodes, got ${toString (builtins.length dist.nodes)}."; + } + { + assertion = dist.enable -> builtins.length dist.volumes >= 4; + message = "services.rustfs.distributed.volumes needs at least 4 drives per node, got ${toString (builtins.length dist.volumes)}."; + } + { + assertion = dist.enable -> builtins.elem dist.localEndpointHost dist.nodes; + message = "services.rustfs.distributed.localEndpointHost is ${dist.localEndpointHost}, which is not one of nodes (${lib.concatStringsSep ", " dist.nodes})."; + } + ]; + users.groups = lib.mkIf (cfg.group == "rustfs") { rustfs = { }; }; @@ -157,8 +235,12 @@ in systemd.tmpfiles.rules = [ "d ${cfg.tlsDirectory} 0750 ${cfg.user} ${cfg.group} -" - ] ++ (map (vol: "d ${vol} 0750 ${cfg.user} ${cfg.group} -") volumesList) - ++ (lib.optional (cfg.logDirectory != null) "d ${cfg.logDirectory} 0750 ${cfg.user} ${cfg.group} -"); + ] + ++ (map (vol: "d ${vol} 0750 ${cfg.user} ${cfg.group} -") localVolumes) + ++ (lib.optional + ( + cfg.logDirectory != null + ) "d ${cfg.logDirectory} 0750 ${cfg.user} ${cfg.group} -"); systemd.services.rustfs = { description = "RustFS Object Storage Server"; @@ -177,9 +259,14 @@ in # Use %d to reference the credentials directory set by LoadCredential RUSTFS_ACCESS_KEY_FILE = "%d/access-key"; RUSTFS_SECRET_KEY_FILE = "%d/secret-key"; - } // lib.optionalAttrs (cfg.logDirectory != null) { + } + // lib.optionalAttrs (cfg.logDirectory != null) { RUSTFS_OBS_LOG_DIRECTORY = cfg.logDirectory; - } // cfg.extraEnvironmentVariables; + } + // lib.optionalAttrs dist.enable { + RUSTFS_LOCAL_ENDPOINT_HOST = dist.localEndpointHost; + } + // cfg.extraEnvironmentVariables; serviceConfig = { User = cfg.user; @@ -239,10 +326,18 @@ in ProtectProc = "invisible"; # Make system directories read-only except for paths we explicitly allow ProtectSystem = "strict"; - # Restrict /proc access - ProcSubset = "pid"; + # Restrict /proc access; multi-drive setups need /proc/mounts for rustfs' + # cross-device mount validation, which "pid" hides. + ProcSubset = if builtins.length localVolumes > 1 then "all" else "pid"; # Restrict network address families to what's needed - RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ]; + # AF_NETLINK: getifaddrs needs it to enumerate local interfaces, which is how + # RustFS decides which endpoints are its own drives rather than a peer's. + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + "AF_UNIX" + "AF_NETLINK" + ]; # Restrict namespaces RestrictNamespaces = true; # Prevent realtime scheduling @@ -252,7 +347,11 @@ in # Restrict to native system calls only SystemCallArchitectures = "native"; # Allow only safe system calls - SystemCallFilter = [ "@system-service" "~@privileged" "~@resources" ]; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "~@resources" + ]; # Prevent memory mapping executable MemoryDenyWriteExecute = true; # Prevent personality changes @@ -261,18 +360,17 @@ in UMask = "0077"; # Grant write access to necessary directories - ReadWritePaths = [ cfg.tlsDirectory ] ++ volumesList - ++ lib.optional (cfg.logDirectory != null) cfg.logDirectory; + ReadWritePaths = [ + cfg.tlsDirectory + ] + ++ localVolumes + ++ lib.optional (cfg.logDirectory != null) cfg.logDirectory; # Logging: Default to systemd journal, optionally write to files StandardOutput = - if cfg.logDirectory != null - then "append:${cfg.logDirectory}/rustfs.log" - else "journal"; + if cfg.logDirectory != null then "append:${cfg.logDirectory}/rustfs.log" else "journal"; StandardError = - if cfg.logDirectory != null - then "append:${cfg.logDirectory}/rustfs-err.log" - else "journal"; + if cfg.logDirectory != null then "append:${cfg.logDirectory}/rustfs-err.log" else "journal"; }; }; }; diff --git a/tests/default.nix b/tests/default.nix new file mode 100644 index 0000000..c013e4a --- /dev/null +++ b/tests/default.nix @@ -0,0 +1,6 @@ +{ pkgs, self }: + +{ + singleNodeMultiDisk = import ./single-node-multi-disk.nix { inherit pkgs self; }; + multiNodeMultiDisk = import ./multi-node-multi-disk.nix { inherit pkgs self; }; +} diff --git a/tests/multi-node-multi-disk.nix b/tests/multi-node-multi-disk.nix new file mode 100644 index 0000000..141e109 --- /dev/null +++ b/tests/multi-node-multi-disk.nix @@ -0,0 +1,112 @@ +{ pkgs, self }: + +let + lib = pkgs.lib; + + hosts = [ "node1" "node2" "node3" "node4" ]; + disks = [ "vdb" "vdc" "vdd" "vde" ]; + + accessKey = "clusteradmin"; + secretKey = "a-strong-random-test-secret"; + + localVolumes = lib.imap0 (i: _: "/mnt/rustfs${toString i}") disks; + + endpoints = lib.concatMap (volume: map (host: "http://${host}:9000${volume}") hosts) localVolumes; + + nodeConfig = name: { lib, ... }: { + imports = [ self.nixosModules.rustfs ]; + + virtualisation.emptyDiskImages = map (_: 1024) disks; + virtualisation.memorySize = 2048; + boot.supportedFilesystems = [ "xfs" ]; + + networking.hostName = name; + networking.firewall.allowedTCPPorts = [ 9000 ]; + + + virtualisation.fileSystems = lib.listToAttrs ( + lib.imap0 + (i: disk: lib.nameValuePair (builtins.elemAt localVolumes i) { + device = "/dev/${disk}"; + fsType = "xfs"; + autoFormat = true; + }) + disks + ); + + services.rustfs = { + enable = true; + package = self.packages.${pkgs.stdenv.hostPlatform.system}.default; + distributed = { + enable = true; + nodes = hosts; + volumes = localVolumes; + port = 9000; + }; + address = "0.0.0.0:9000"; + consoleEnable = false; + accessKeyFile = "/etc/rustfs-access-key"; + secretKeyFile = "/etc/rustfs-secret-key"; + }; + + # Non-default credentials: RustFS derives the inter-node RPC secret from them + # and refuses to derive one from rustfsadmin/rustfsadmin. + environment.etc."rustfs-access-key".text = accessKey; + environment.etc."rustfs-secret-key".text = secretKey; + + environment.systemPackages = [ pkgs.awscli2 ]; + }; +in +pkgs.testers.runNixOSTest { + name = "rustfs-multi-node-multi-disk"; + + nodes = lib.listToAttrs (map (host: lib.nameValuePair host (nodeConfig host)) hosts); + + testScript = '' + start_all() + + nodes = [${lib.concatStringsSep ", " hosts}] + + for node in nodes: + node.wait_for_unit("rustfs.service") + node.wait_for_open_port(9000) + + node1.succeed( + "systemctl show -p Environment rustfs.service | " + "grep -q 'RUSTFS_VOLUMES=${lib.concatStringsSep " " endpoints}'" + ) + + # Each node must claim its own drives; 0 local endpoints means every disk + # access becomes an RPC to itself, which then fails signature verification. + for node in nodes: + node.wait_until_succeeds( + "journalctl -u rustfs.service | grep -q " + "'\"total_endpoints\":${toString (builtins.length endpoints)},\"local_endpoints\":${toString (builtins.length localVolumes)}'", + timeout=60, + ) + + node1.fail("systemctl show -p ReadWritePaths rustfs.service | grep -q 'http://'") + node1.succeed("test ! -e '/http:'") + for volume in [${lib.concatStringsSep ", " (map (v: "\"${v}\"") localVolumes)}]: + node1.succeed(f"test -d {volume}") + + aws = ( + "AWS_ACCESS_KEY_ID=${accessKey} AWS_SECRET_ACCESS_KEY=${secretKey} " + "AWS_DEFAULT_REGION=us-east-1 aws --endpoint-url http://127.0.0.1:9000" + ) + + node1.wait_until_succeeds(aws + " s3 ls", timeout=300) + + node1.succeed(aws + " s3 mb s3://testbucket") + node1.succeed("echo distributed > /tmp/obj.txt") + node1.succeed(aws + " s3 cp /tmp/obj.txt s3://testbucket/obj.txt") + + for node in nodes[1:]: + node.wait_until_succeeds(aws + " s3 ls s3://testbucket/", timeout=120) + node.succeed(aws + " s3 cp s3://testbucket/obj.txt /tmp/roundtrip.txt") + node.succeed("grep -q distributed /tmp/roundtrip.txt") + + for node in nodes: + node.succeed("systemctl is-active rustfs.service") + ''; +} diff --git a/tests/single-node-multi-disk.nix b/tests/single-node-multi-disk.nix new file mode 100644 index 0000000..e6d4700 --- /dev/null +++ b/tests/single-node-multi-disk.nix @@ -0,0 +1,68 @@ +{ pkgs, self }: + +let + disks = [ "vdb" "vdc" "vdd" "vde" ]; + volumes = pkgs.lib.imap0 (i: _: "/mnt/rustfs${toString i}") disks; +in +pkgs.testers.runNixOSTest { + name = "rustfs-single-node-multi-disk"; + + nodes.machine = { lib, ... }: { + imports = [ self.nixosModules.rustfs ]; + + virtualisation.emptyDiskImages = map (_: 1024) disks; + virtualisation.memorySize = 2048; + boot.supportedFilesystems = [ "xfs" ]; + + virtualisation.fileSystems = lib.listToAttrs ( + lib.imap0 + (i: disk: lib.nameValuePair (builtins.elemAt volumes i) { + device = "/dev/${disk}"; + fsType = "xfs"; + autoFormat = true; + }) + disks + ); + + services.rustfs = { + enable = true; + package = self.packages.${pkgs.stdenv.hostPlatform.system}.default; + inherit volumes; + address = "127.0.0.1:9000"; + consoleEnable = false; + accessKeyFile = "/etc/rustfs-access-key"; + secretKeyFile = "/etc/rustfs-secret-key"; + }; + + environment.etc."rustfs-access-key".text = "rustfsadmin"; + environment.etc."rustfs-secret-key".text = "rustfsadmin"; + environment.systemPackages = [ pkgs.awscli2 ]; + }; + + testScript = '' + machine.wait_for_unit("rustfs.service") + machine.wait_for_open_port(9000) + + machine.succeed( + "systemctl show -p Environment rustfs.service | " + "grep -q 'RUSTFS_VOLUMES=${builtins.concatStringsSep " " volumes}'" + ) + + aws = ( + "AWS_ACCESS_KEY_ID=rustfsadmin AWS_SECRET_ACCESS_KEY=rustfsadmin " + "AWS_DEFAULT_REGION=us-east-1 aws --endpoint-url http://127.0.0.1:9000" + ) + machine.wait_until_succeeds(aws + " s3 ls", timeout=120) + + machine.succeed(aws + " s3 mb s3://testbucket") + machine.succeed("echo hello > /tmp/obj.txt") + machine.succeed(aws + " s3 cp /tmp/obj.txt s3://testbucket/obj.txt") + machine.succeed(aws + " s3 cp s3://testbucket/obj.txt /tmp/roundtrip.txt") + machine.succeed("grep -q hello /tmp/roundtrip.txt") + + for volume in [${builtins.concatStringsSep ", " (map (v: "\"${v}\"") volumes)}]: + machine.succeed(f"find {volume} -name 'xl.meta' | grep -q .") + + machine.succeed("systemctl is-active rustfs.service") + ''; +}