fix: private IPFS swarm nodes never peer, breaking shared storage

The pinned ipfs/go-ipfs:v0.10.0 image has a private-network (pnet)
connection bug where swarm connections drop immediately after
handshake, so org IPFS nodes never peer with each other. This causes
"failed to bootstrap (no peers found)" and shared storage downloads
to time out fetching content pinned only on another member's node.

Bump to ipfs/kubo:v0.42.0, and since modern Kubo refuses to start in
private-network mode with its default AutoConf/public bootstrap
config, add a container-init.d script that disables it for
private-mode stacks.

mDNS auto-discovery, which worked locally, proved unreliable on
native Linux Docker bridge networks - members' nodes never found each
other there. Add an explicit peering step after first-time-setup that
queries each member's real PeerID via its own API and calls
swarm/peering/add against every other member, so nodes connect (and
persistently reconnect) regardless of whether mDNS works in a given
environment.

Signed-off-by: Enrique Lacal <enrique.lacal@kaleido.io>

Author: EnriqueL8

internal/constants/constants.go

@@ -23,7 +23,7 @@ import (
 
 var StacksDir = checkHome()
 var FireFlyCoreImageName = "ghcr.io/hyperledger-firefly/firefly"
-var IPFSImageName = "ipfs/go-ipfs:v0.10.0"
+var IPFSImageName = "ipfs/kubo:v0.42.0"
 var PostgresImageName = "postgres"
 var PrometheusImageName = "prom/prometheus"
 var SandboxImageName = "ghcr.io/hyperledger-firefly/sandbox:latest"

internal/docker/docker_config.go

@@ -167,6 +167,11 @@ func CreateDockerCompose(s *types.Stack) *DockerComposeConfig {
 				"LIBP2P_FORCE_PNET": "1",
 			},
 			)
+			// Kubo's AutoConf/public-network defaults are incompatible with a
+			// private swarm and prevent peering with other members unless
+			// disabled via a container-init.d script - see ipfs_config.go.
+			sharedStorage.Volumes = append(sharedStorage.Volumes, fmt.Sprintf("ipfs_init_%s:/container-init.d", member.ID))
+			compose.Volumes[fmt.Sprintf("ipfs_init_%s", member.ID)] = struct{}{}
 		} else {
 			sharedStorage.Environment = s.EnvironmentVars
 		}

internal/stacks/ipfs_config.go

@@ -29,3 +29,20 @@ func GenerateSwarmKey() (string, error) {
 	hexKey := hex.EncodeToString(key)
 	return "/key/swarm/psk/1.0.0/\n/base16/\n" + hexKey, nil
 }
+
+// GenerateIPFSPrivateNetInitScript returns a container-init.d script that
+// disables Kubo's AutoConf/public-network defaults, which are incompatible
+// with a private swarm (swarm.key / LIBP2P_FORCE_PNET) and otherwise prevent
+// the daemon from starting or from ever peering with other members.
+func GenerateIPFSPrivateNetInitScript() string {
+	return `#!/bin/sh
+ipfs config --json AutoConf.Enabled false
+ipfs config --json Bootstrap '[]'
+ipfs config --json DNS.Resolvers '{}'
+ipfs config --json Routing.DelegatedRouters '[]'
+ipfs config --json Ipns.DelegatedPublishers '[]'
+ipfs config Routing.Type dht
+ipfs config --json AutoTLS.Enabled false
+ipfs config --json Swarm.Transports.Network.Websocket false
+`
+}

internal/stacks/stack_manager.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"net/http"
 	"os"
 	"os/exec"
 	"path"
@@ -505,6 +506,13 @@ func (s *StackManager) writeConfig(options *types.InitOptions) error {
 		}
 	}
 
+	if s.Stack.IPFSMode.Equals(types.IPFSModePrivate) {
+		initScript := GenerateIPFSPrivateNetInitScript()
+		if err := os.WriteFile(path.Join(s.Stack.InitDir, "config", "ipfs_privatenet_init.sh"), []byte(initScript), 0755); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -566,6 +574,61 @@ func (s *StackManager) copyDataExchangeConfigToVolumes() error {
 	return nil
 }
 
+func (s *StackManager) copyIPFSInitScriptToVolumes() error {
+	if !s.Stack.IPFSMode.Equals(types.IPFSModePrivate) {
+		return nil
+	}
+	configDir := filepath.Join(s.Stack.RuntimeDir, "config")
+	scriptPath := path.Join(configDir, "ipfs_privatenet_init.sh")
+	for _, member := range s.Stack.Members {
+		volumeName := fmt.Sprintf("%s_ipfs_init_%s", s.Stack.Name, member.ID)
+		if err := docker.CopyFileToVolume(s.ctx, volumeName, scriptPath, "/privatenet-init.sh"); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// peerIPFSNodes explicitly peers every private-mode IPFS node with every
+// other member's node. mDNS auto-discovery has proven unreliable across
+// different Docker networking environments (it connected nodes on Docker
+// Desktop but not on a native Linux Docker bridge network, such as GitHub
+// Actions runners use), so without this, members' IPFS nodes may never
+// connect to each other and shared storage downloads will hang/time out.
+func (s *StackManager) peerIPFSNodes() error {
+	if !s.Stack.IPFSMode.Equals(types.IPFSModePrivate) || len(s.Stack.Members) < 2 {
+		return nil
+	}
+
+	type ipfsIDResponse struct {
+		ID string `json:"ID"`
+	}
+
+	peerIDs := make(map[string]string, len(s.Stack.Members))
+	for _, member := range s.Stack.Members {
+		var idResp ipfsIDResponse
+		url := fmt.Sprintf("http://127.0.0.1:%d/api/v0/id", member.ExposedIPFSApiPort)
+		if err := core.RequestWithRetry(s.ctx, http.MethodPost, url, nil, &idResp); err != nil {
+			return fmt.Errorf("failed to get IPFS peer ID for member %s: %w", member.ID, err)
+		}
+		peerIDs[member.ID] = idResp.ID
+	}
+
+	for _, member := range s.Stack.Members {
+		for _, other := range s.Stack.Members {
+			if member.ID == other.ID {
+				continue
+			}
+			addr := fmt.Sprintf("/dns4/ipfs_%s/tcp/4001/p2p/%s", other.ID, peerIDs[other.ID])
+			url := fmt.Sprintf("http://127.0.0.1:%d/api/v0/swarm/peering/add?arg=%s", member.ExposedIPFSApiPort, addr)
+			if err := core.RequestWithRetry(s.ctx, http.MethodPost, url, nil, nil); err != nil {
+				return fmt.Errorf("failed to peer IPFS node %s with %s: %w", member.ID, other.ID, err)
+			}
+		}
+	}
+	return nil
+}
+
 func (s *StackManager) createMember(id string, index int, options *types.InitOptions, external bool) (*types.Organization, error) {
 	serviceBase := options.ServicesBasePort + (index * 100)
 	ptmBase := options.PtmBasePort + (index * 10)
@@ -908,6 +971,10 @@ func (s *StackManager) runFirstTimeSetup(options *types.StartOptions) (messages
 		return messages, err
 	}
 
+	if err := s.copyIPFSInitScriptToVolumes(); err != nil {
+		return messages, err
+	}
+
 	pullOptions := &types.PullOptions{
 		Retries: 2,
 	}
@@ -919,6 +986,10 @@ func (s *StackManager) runFirstTimeSetup(options *types.StartOptions) (messages
 		return messages, err
 	}
 
+	if err := s.peerIPFSNodes(); err != nil {
+		return messages, err
+	}
+
 	for i, tp := range s.tokenProviders {
 		if !s.Stack.DisableTokenFactories {
 			result, err := tp.DeploySmartContracts(i)