From 037d09c32e453bc1ca68830f82090e6655716536 Mon Sep 17 00:00:00 2001 From: Enrique Lacal Date: Thu, 23 Jul 2026 15:49:50 +0100 Subject: [PATCH 1/2] fix: private IPFS swarm nodes never peer, breaking shared storage The pinned ipfs/go-ipfs:v0.10.0 image has a private-network (pnet) connection bug where swarm connections drop immediately after handshake, so org IPFS nodes never peer with each other. This causes "failed to bootstrap (no peers found)" and shared storage downloads to time out fetching content pinned only on another member's node. Bump to ipfs/kubo:v0.42.0, and since modern Kubo refuses to start in private-network mode with its default AutoConf/public bootstrap config, add a container-init.d script that disables it for private-mode stacks. mDNS auto-discovery, which worked locally, proved unreliable on native Linux Docker bridge networks - members' nodes never found each other there. Add an explicit peering step after first-time-setup that queries each member's real PeerID via its own API and calls swarm/peering/add against every other member, so nodes connect (and persistently reconnect) regardless of whether mDNS works in a given environment. Signed-off-by: Enrique Lacal --- internal/constants/constants.go | 2 +- internal/docker/docker_config.go | 5 +++ internal/stacks/ipfs_config.go | 17 ++++++++ internal/stacks/stack_manager.go | 71 ++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 1 deletion(-) diff --git a/internal/constants/constants.go b/internal/constants/constants.go index ac2d7bbf..21f47066 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -23,7 +23,7 @@ import ( var StacksDir = checkHome() var FireFlyCoreImageName = "ghcr.io/hyperledger-firefly/firefly" -var IPFSImageName = "ipfs/go-ipfs:v0.10.0" +var IPFSImageName = "ipfs/kubo:v0.42.0" var PostgresImageName = "postgres" var PrometheusImageName = "prom/prometheus" var SandboxImageName = "ghcr.io/hyperledger-firefly/sandbox:latest" diff --git a/internal/docker/docker_config.go b/internal/docker/docker_config.go index 97088d5a..383c4d17 100644 --- a/internal/docker/docker_config.go +++ b/internal/docker/docker_config.go @@ -167,6 +167,11 @@ func CreateDockerCompose(s *types.Stack) *DockerComposeConfig { "LIBP2P_FORCE_PNET": "1", }, ) + // Kubo's AutoConf/public-network defaults are incompatible with a + // private swarm and prevent peering with other members unless + // disabled via a container-init.d script - see ipfs_config.go. + sharedStorage.Volumes = append(sharedStorage.Volumes, fmt.Sprintf("ipfs_init_%s:/container-init.d", member.ID)) + compose.Volumes[fmt.Sprintf("ipfs_init_%s", member.ID)] = struct{}{} } else { sharedStorage.Environment = s.EnvironmentVars } diff --git a/internal/stacks/ipfs_config.go b/internal/stacks/ipfs_config.go index a63f006c..e05f8541 100644 --- a/internal/stacks/ipfs_config.go +++ b/internal/stacks/ipfs_config.go @@ -29,3 +29,20 @@ func GenerateSwarmKey() (string, error) { hexKey := hex.EncodeToString(key) return "/key/swarm/psk/1.0.0/\n/base16/\n" + hexKey, nil } + +// GenerateIPFSPrivateNetInitScript returns a container-init.d script that +// disables Kubo's AutoConf/public-network defaults, which are incompatible +// with a private swarm (swarm.key / LIBP2P_FORCE_PNET) and otherwise prevent +// the daemon from starting or from ever peering with other members. +func GenerateIPFSPrivateNetInitScript() string { + return `#!/bin/sh +ipfs config --json AutoConf.Enabled false +ipfs config --json Bootstrap '[]' +ipfs config --json DNS.Resolvers '{}' +ipfs config --json Routing.DelegatedRouters '[]' +ipfs config --json Ipns.DelegatedPublishers '[]' +ipfs config Routing.Type dht +ipfs config --json AutoTLS.Enabled false +ipfs config --json Swarm.Transports.Network.Websocket false +` +} diff --git a/internal/stacks/stack_manager.go b/internal/stacks/stack_manager.go index c8bfb7f6..5907b26e 100644 --- a/internal/stacks/stack_manager.go +++ b/internal/stacks/stack_manager.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "net" + "net/http" "os" "os/exec" "path" @@ -505,6 +506,13 @@ func (s *StackManager) writeConfig(options *types.InitOptions) error { } } + if s.Stack.IPFSMode.Equals(types.IPFSModePrivate) { + initScript := GenerateIPFSPrivateNetInitScript() + if err := os.WriteFile(path.Join(s.Stack.InitDir, "config", "ipfs_privatenet_init.sh"), []byte(initScript), 0755); err != nil { + return err + } + } + return nil } @@ -566,6 +574,61 @@ func (s *StackManager) copyDataExchangeConfigToVolumes() error { return nil } +func (s *StackManager) copyIPFSInitScriptToVolumes() error { + if !s.Stack.IPFSMode.Equals(types.IPFSModePrivate) { + return nil + } + configDir := filepath.Join(s.Stack.RuntimeDir, "config") + scriptPath := path.Join(configDir, "ipfs_privatenet_init.sh") + for _, member := range s.Stack.Members { + volumeName := fmt.Sprintf("%s_ipfs_init_%s", s.Stack.Name, member.ID) + if err := docker.CopyFileToVolume(s.ctx, volumeName, scriptPath, "/privatenet-init.sh"); err != nil { + return err + } + } + return nil +} + +// peerIPFSNodes explicitly peers every private-mode IPFS node with every +// other member's node. mDNS auto-discovery has proven unreliable across +// different Docker networking environments (it connected nodes on Docker +// Desktop but not on a native Linux Docker bridge network, such as GitHub +// Actions runners use), so without this, members' IPFS nodes may never +// connect to each other and shared storage downloads will hang/time out. +func (s *StackManager) peerIPFSNodes() error { + if !s.Stack.IPFSMode.Equals(types.IPFSModePrivate) || len(s.Stack.Members) < 2 { + return nil + } + + type ipfsIDResponse struct { + ID string `json:"ID"` + } + + peerIDs := make(map[string]string, len(s.Stack.Members)) + for _, member := range s.Stack.Members { + var idResp ipfsIDResponse + url := fmt.Sprintf("http://127.0.0.1:%d/api/v0/id", member.ExposedIPFSApiPort) + if err := core.RequestWithRetry(s.ctx, http.MethodPost, url, nil, &idResp); err != nil { + return fmt.Errorf("failed to get IPFS peer ID for member %s: %w", member.ID, err) + } + peerIDs[member.ID] = idResp.ID + } + + for _, member := range s.Stack.Members { + for _, other := range s.Stack.Members { + if member.ID == other.ID { + continue + } + addr := fmt.Sprintf("/dns4/ipfs_%s/tcp/4001/p2p/%s", other.ID, peerIDs[other.ID]) + url := fmt.Sprintf("http://127.0.0.1:%d/api/v0/swarm/peering/add?arg=%s", member.ExposedIPFSApiPort, addr) + if err := core.RequestWithRetry(s.ctx, http.MethodPost, url, nil, nil); err != nil { + return fmt.Errorf("failed to peer IPFS node %s with %s: %w", member.ID, other.ID, err) + } + } + } + return nil +} + func (s *StackManager) createMember(id string, index int, options *types.InitOptions, external bool) (*types.Organization, error) { serviceBase := options.ServicesBasePort + (index * 100) ptmBase := options.PtmBasePort + (index * 10) @@ -908,6 +971,10 @@ func (s *StackManager) runFirstTimeSetup(options *types.StartOptions) (messages return messages, err } + if err := s.copyIPFSInitScriptToVolumes(); err != nil { + return messages, err + } + pullOptions := &types.PullOptions{ Retries: 2, } @@ -919,6 +986,10 @@ func (s *StackManager) runFirstTimeSetup(options *types.StartOptions) (messages return messages, err } + if err := s.peerIPFSNodes(); err != nil { + return messages, err + } + for i, tp := range s.tokenProviders { if !s.Stack.DisableTokenFactories { result, err := tp.DeploySmartContracts(i) From 61fef2f278d9201eecd6916d28f2b958fbfb43ac Mon Sep 17 00:00:00 2001 From: Enrique Lacal Date: Fri, 24 Jul 2026 16:05:50 +0100 Subject: [PATCH 2/2] Peer the nodes after restart Signed-off-by: Enrique Lacal --- internal/stacks/stack_manager.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/stacks/stack_manager.go b/internal/stacks/stack_manager.go index 5907b26e..1f23492d 100644 --- a/internal/stacks/stack_manager.go +++ b/internal/stacks/stack_manager.go @@ -986,10 +986,6 @@ func (s *StackManager) runFirstTimeSetup(options *types.StartOptions) (messages return messages, err } - if err := s.peerIPFSNodes(); err != nil { - return messages, err - } - for i, tp := range s.tokenProviders { if !s.Stack.DisableTokenFactories { result, err := tp.DeploySmartContracts(i) @@ -1106,6 +1102,13 @@ func (s *StackManager) runFirstTimeSetup(options *types.StartOptions) (messages return messages, err } + // Peer the IPFS nodes on the finalized containers, after the config + // restart above, so peering is applied to the containers that will keep + // running rather than relying on it surviving the restart. + if err := s.peerIPFSNodes(); err != nil { + return messages, err + } + if s.Stack.MultipartyEnabled { if s.Stack.ContractAddress == "" { s.Log.Info("registering FireFly identities")