fix: IPFS peering never persists, and bitswap hangs even when peered

Two separate bugs were still causing shared storage downloads to
hang/time out intermittently even after the earlier peering fix
(#359):

1. swarm/peering/add returns success but never actually writes to
   the on-disk Peering.Peers config - verified directly by checking
   the config immediately after calling it. It only registers the
   peer with the running daemon's in-memory peering service for that
   session. Since first-time setup restarts the IPFS containers
   partway through (to apply finalized config), any peering set up
   before that restart was silently lost, leaving nodes to fall back
   on unreliable mDNS. Switched to POST /api/v0/config to persist
   Peering.Peers for real; verified it survives a full container
   restart with both nodes reconnecting automatically.

2. Even once genuinely peered, content fetches still hang: this is a
   long-standing upstream Kubo/bitswap bug (ipfs/kubo#8346, open
   since 2021) where in a small private network, bitswap discovers a
   provider via the DHT but never sends it a WANT message if the
   swarm connection to that peer already existed - reproduced
   directly (bitswap stat showed 0 partners despite an active,
   bitswap-protocol-negotiated swarm connection). Routing.Type is now
   set to "none" instead of "dht": with no DHT to depend on, bitswap
   has no choice but to broadcast wants directly to its connected
   peers, which is all a small private swarm needs anyway. Verified
   5/5 sequential cross-node fetches plus the HTTP gateway path all
   succeed reliably with this change.

Signed-off-by: Enrique Lacal <enrique.lacal@kaleido.io>

Author: EnriqueL8

internal/stacks/ipfs_config.go

@@ -34,14 +34,22 @@ func GenerateSwarmKey() (string, error) {
 // disables Kubo's AutoConf/public-network defaults, which are incompatible
 // with a private swarm (swarm.key / LIBP2P_FORCE_PNET) and otherwise prevent
 // the daemon from starting or from ever peering with other members.
+//
+// Routing.Type is set to "none" rather than "dht": a small private swarm hits
+// a long-standing upstream bug (https://github.com/ipfs/kubo/issues/8346)
+// where bitswap finds a provider via the DHT but never sends it a WANT
+// message if the swarm connection to that peer already existed, so content
+// fetches hang even though the peers are connected. With routing disabled,
+// bitswap has no choice but to broadcast wants directly to its connected
+// peers, which is all a 2+ node private swarm actually needs.
 func GenerateIPFSPrivateNetInitScript() string {
 	return `#!/bin/sh
 ipfs config --json AutoConf.Enabled false
 ipfs config --json Bootstrap '[]'
 ipfs config --json DNS.Resolvers '{}'
 ipfs config --json Routing.DelegatedRouters '[]'
 ipfs config --json Ipns.DelegatedPublishers '[]'
-ipfs config Routing.Type dht
+ipfs config Routing.Type none
 ipfs config --json AutoTLS.Enabled false
 ipfs config --json Swarm.Transports.Network.Websocket false
 `

internal/stacks/stack_manager.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
 	"path"
@@ -614,24 +615,46 @@ func (s *StackManager) peerIPFSNodes() error {
 		peerIDs[member.ID] = idResp.ID
 	}
 
+	type peeringPeer struct {
+		ID    string   `json:"ID"`
+		Addrs []string `json:"Addrs"`
+	}
+
 	for _, member := range s.Stack.Members {
+		peers := []peeringPeer{}
 		for _, other := range s.Stack.Members {
 			if member.ID == other.ID {
 				continue
 			}
-			addr := fmt.Sprintf("/dns4/ipfs_%s/tcp/4001/p2p/%s", other.ID, peerIDs[other.ID])
+			peers = append(peers, peeringPeer{
+				ID:    peerIDs[other.ID],
+				Addrs: []string{fmt.Sprintf("/dns4/ipfs_%s/tcp/4001", other.ID)},
+			})
+		}
+		peersJSON, err := json.Marshal(peers)
+		if err != nil {
+			return err
+		}
 
-			// swarm/peering/add only registers the peer with Kubo's background
-			// reconnect service - it returns success even when the peer is
-			// unreachable, so it does not confirm a connection was made.
-			peeringURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/swarm/peering/add?arg=%s", member.ExposedIPFSApiPort, addr)
-			if err := core.RequestWithRetry(s.ctx, http.MethodPost, peeringURL, nil, nil); err != nil {
-				return fmt.Errorf("failed to peer IPFS node %s with %s: %w", member.ID, other.ID, err)
-			}
+		// swarm/peering/add only registers the peer with the *running*
+		// daemon's in-memory peering service and returns success even when
+		// the peer is unreachable - it does not persist to Peering.Peers in
+		// the on-disk config, so the pairing is lost on the next restart.
+		// Writing through /api/v0/config persists it for real, and Kubo
+		// picks up the change live (no restart needed).
+		configURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/config?arg=Peering.Peers&arg=%s&json=true", member.ExposedIPFSApiPort, url.QueryEscape(string(peersJSON)))
+		if err := core.RequestWithRetry(s.ctx, http.MethodPost, configURL, nil, nil); err != nil {
+			return fmt.Errorf("failed to persist IPFS peering config for member %s: %w", member.ID, err)
+		}
 
+		for _, other := range s.Stack.Members {
+			if member.ID == other.ID {
+				continue
+			}
+			addr := fmt.Sprintf("/dns4/ipfs_%s/tcp/4001/p2p/%s", other.ID, peerIDs[other.ID])
 			// swarm/connect dials synchronously and errors if the connection
 			// fails, so retrying it confirms the nodes are actually connected
-			// rather than just registered to reconnect in the background.
+			// now rather than waiting on the peering service's own timing.
 			connectURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/swarm/connect?arg=%s", member.ExposedIPFSApiPort, addr)
 			if err := core.RequestWithRetry(s.ctx, http.MethodPost, connectURL, nil, nil); err != nil {
 				return fmt.Errorf("failed to connect IPFS node %s to %s: %w", member.ID, other.ID, err)