fix: explicitly peer private-mode IPFS nodes after startup

mDNS auto-discovery has proven unreliable across different Docker
networking environments - verified working on Docker Desktop but not
on a native Linux Docker bridge network (e.g. GitHub Actions runners),
where sibling IPFS nodes never connected to each other despite the
earlier AutoConf/Bootstrap fix, causing shared storage downloads to
hang/time out again.

Query each member's real PeerID via its own IPFS API after containers
start, then call swarm/peering/add on every other member so they
connect to (and persistently reconnect to) each other, regardless of
whether mDNS works in a given environment.

Signed-off-by: Enrique Lacal <enrique.lacal@kaleido.io>

Author: EnriqueL8

.github/workflows/debug-ipfs-mdns.yml

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"net/http"
 	"os"
 	"os/exec"
 	"path"
@@ -588,6 +589,46 @@ func (s *StackManager) copyIPFSInitScriptToVolumes() error {
 	return nil
 }
 
+// peerIPFSNodes explicitly peers every private-mode IPFS node with every
+// other member's node. mDNS auto-discovery has proven unreliable across
+// different Docker networking environments (it connected nodes on Docker
+// Desktop but not on a native Linux Docker bridge network, such as GitHub
+// Actions runners use), so without this, members' IPFS nodes may never
+// connect to each other and shared storage downloads will hang/time out.
+func (s *StackManager) peerIPFSNodes() error {
+	if !s.Stack.IPFSMode.Equals(types.IPFSModePrivate) || len(s.Stack.Members) < 2 {
+		return nil
+	}
+
+	type ipfsIDResponse struct {
+		ID string `json:"ID"`
+	}
+
+	peerIDs := make(map[string]string, len(s.Stack.Members))
+	for _, member := range s.Stack.Members {
+		var idResp ipfsIDResponse
+		url := fmt.Sprintf("http://127.0.0.1:%d/api/v0/id", member.ExposedIPFSApiPort)
+		if err := core.RequestWithRetry(s.ctx, http.MethodPost, url, nil, &idResp); err != nil {
+			return fmt.Errorf("failed to get IPFS peer ID for member %s: %w", member.ID, err)
+		}
+		peerIDs[member.ID] = idResp.ID
+	}
+
+	for _, member := range s.Stack.Members {
+		for _, other := range s.Stack.Members {
+			if member.ID == other.ID {
+				continue
+			}
+			addr := fmt.Sprintf("/dns4/ipfs_%s/tcp/4001/p2p/%s", other.ID, peerIDs[other.ID])
+			url := fmt.Sprintf("http://127.0.0.1:%d/api/v0/swarm/peering/add?arg=%s", member.ExposedIPFSApiPort, addr)
+			if err := core.RequestWithRetry(s.ctx, http.MethodPost, url, nil, nil); err != nil {
+				return fmt.Errorf("failed to peer IPFS node %s with %s: %w", member.ID, other.ID, err)
+			}
+		}
+	}
+	return nil
+}
+
 func (s *StackManager) createMember(id string, index int, options *types.InitOptions, external bool) (*types.Organization, error) {
 	serviceBase := options.ServicesBasePort + (index * 100)
 	ptmBase := options.PtmBasePort + (index * 10)
@@ -945,6 +986,10 @@ func (s *StackManager) runFirstTimeSetup(options *types.StartOptions) (messages
 		return messages, err
 	}
 
+	if err := s.peerIPFSNodes(); err != nil {
+		return messages, err
+	}
+
 	for i, tp := range s.tokenProviders {
 		if !s.Stack.DisableTokenFactories {
 			result, err := tp.DeploySmartContracts(i)