Improve nodes startup in systests (#6706)

## Motivation

I observed bootnodes being killed by the k8s startup probe in systests:
```
 Normal   Killing    38s                kubelet            Container smesher failed startup probe, will be restarted
```
This happens because bootnodes are deployed before poets and they try to reach poets, retry few times and the startup probe times out.

Author: poszu

node/node.go

@@ -199,6 +199,12 @@ func GetCommand() *cobra.Command {
 
 			// This blocks until the context is finished or until an error is produced
 			err = app.Start(ctx)
+			if err != nil {
+				app.log.With().Error("app failed", log.Err(err))
+			} else {
+				app.log.With().Info("app stopped", log.Err(ctx.Err()))
+			}
+
 			cleanupCtx, cleanupCancel := context.WithTimeout(
 				context.Background(),
 				30*time.Second,

systest/cluster/cluster.go

@@ -173,17 +173,6 @@ func Default(cctx *testcontext.Context, opts ...Opt) (*Cluster, error) {
 		zap.Int("remote", cctx.RemoteSize),
 	)
 
-	keys := make([]ed25519.PrivateKey, cctx.ClusterSize)
-	for i := range keys {
-		keys[i] = cl.accounts.Private(i)
-	}
-
-	if err := cl.AddBootnodes(cctx, cctx.BootnodeSize); err != nil {
-		return nil, err
-	}
-	if err := cl.AddBootstrappers(cctx); err != nil {
-		return nil, err
-	}
 	pubkey, privkey, err := ed25519.GenerateKey(nil)
 	if err != nil {
 		return nil, fmt.Errorf("generating keys for certifier: %w", err)
@@ -198,6 +187,18 @@ func Default(cctx *testcontext.Context, opts ...Opt) (*Cluster, error) {
 		return nil, err
 	}
 
+	keys := make([]ed25519.PrivateKey, cctx.ClusterSize)
+	for i := range keys {
+		keys[i] = cl.accounts.Private(i)
+	}
+
+	if err := cl.AddBootnodes(cctx, cctx.BootnodeSize); err != nil {
+		return nil, err
+	}
+	if err := cl.AddBootstrappers(cctx); err != nil {
+		return nil, err
+	}
+
 	smesherKeys := keys[cctx.BootnodeSize : cctx.BootnodeSize+smeshers]
 	if err := cl.AddSmeshers(cctx, smeshers, WithSmeshers(smesherKeys)); err != nil {
 		return nil, err

systest/cluster/nodes.go

@@ -902,9 +902,11 @@ func deployNode(
 				WithLimits(smesherResources.Get(ctx.Parameters).Limits),
 			).
 			WithStartupProbe(
-				corev1.Probe().WithTCPSocket(
-					corev1.TCPSocketAction().WithPort(intstr.FromInt32(9092)),
-				).WithInitialDelaySeconds(10).WithPeriodSeconds(10),
+				corev1.Probe().
+					WithTCPSocket(corev1.TCPSocketAction().WithPort(intstr.FromInt32(9092))).
+					WithInitialDelaySeconds(10).
+					WithPeriodSeconds(5).
+					WithFailureThreshold(12),
 			).
 			WithEnv(
 				corev1.EnvVar().WithName("GOMAXPROCS").WithValue("4"),

systest/tests/distributed_post_verification_test.go

@@ -77,9 +77,9 @@ func TestPostMalfeasanceProof(t *testing.T) {
 	ctx.PoetSize = 1 // one poet guarantees everybody gets the same proof
 	ctx.ClusterSize = 8
 	cl := cluster.New(ctx, cluster.WithKeys(10))
+	require.NoError(t, cl.AddPoets(ctx))
 	require.NoError(t, cl.AddBootnodes(ctx, 1))
 	require.NoError(t, cl.AddBootstrappers(ctx))
-	require.NoError(t, cl.AddPoets(ctx))
 	require.NoError(t, cl.AddSmeshers(ctx, ctx.ClusterSize-cl.Total(), cluster.WithFlags(cluster.PostK3(1))))
 
 	logger := ctx.Log.Desugar().WithOptions(zap.IncreaseLevel(zap.InfoLevel), zap.WithCaller(false))

systest/tests/equivocation_test.go

@@ -29,9 +29,9 @@ func TestEquivocation(t *testing.T) {
 
 	const bootnodes = 2
 	cl := cluster.New(cctx, cluster.WithKeys(cctx.ClusterSize))
+	require.NoError(t, cl.AddPoets(cctx))
 	require.NoError(t, cl.AddBootnodes(cctx, bootnodes))
 	require.NoError(t, cl.AddBootstrappers(cctx))
-	require.NoError(t, cl.AddPoets(cctx))
 
 	smeshers := cctx.ClusterSize - cl.Total()
 	honest := int(float64(smeshers) * 0.6)

systest/tests/poets_test.go

@@ -131,9 +131,9 @@ func TestNodesUsingDifferentPoets(t *testing.T) {
 	}
 
 	cl := cluster.New(tctx, cluster.WithKeys(tctx.ClusterSize))
+	require.NoError(t, cl.AddPoets(tctx))
 	require.NoError(t, cl.AddBootnodes(tctx, 2))
 	require.NoError(t, cl.AddBootstrappers(tctx))
-	require.NoError(t, cl.AddPoets(tctx))
 
 	for i := 0; i < tctx.ClusterSize-2; i++ {
 		poetId := i % tctx.PoetSize
@@ -215,15 +215,12 @@ func TestRegisteringInPoetWithPowAndCert(t *testing.T) {
 	tctx := testcontext.New(t)
 
 	cl := cluster.New(tctx, cluster.WithKeys(10))
-	require.NoError(t, cl.AddBootnodes(tctx, 2))
-	require.NoError(t, cl.AddBootstrappers(tctx))
 
-	pubkey, privkey, err := ed25519.GenerateKey(nil)
-	require.NoError(t, err)
-	require.NoError(t, cl.AddCertifier(tctx, base64.StdEncoding.EncodeToString(privkey.Seed())))
 	// First poet supports PoW only (legacy)
 	require.NoError(t, cl.AddPoet(tctx))
 	// Second poet supports certs
+	pubkey, privkey, err := ed25519.GenerateKey(nil)
+	require.NoError(t, err)
 	require.NoError(
 		t,
 		cl.AddPoet(
@@ -232,6 +229,10 @@ func TestRegisteringInPoetWithPowAndCert(t *testing.T) {
 			cluster.PoetCertifierPubkey(base64.StdEncoding.EncodeToString(pubkey)),
 		),
 	)
+
+	require.NoError(t, cl.AddBootnodes(tctx, 2))
+	require.NoError(t, cl.AddBootstrappers(tctx))
+	require.NoError(t, cl.AddCertifier(tctx, base64.StdEncoding.EncodeToString(privkey.Seed())))
 	require.NoError(t, cl.AddSmeshers(tctx, tctx.ClusterSize-2))
 	require.NoError(t, cl.WaitAll(tctx))