ci: display port usage information

elhimov · elhimov · commit baf83a627a34 · 2025-09-22T22:56:56.000+03:00
It might help to identify the reason of flaky test
`test_replicaset_bootstrap_cartridge_app_second_bootstrap`
when/if it fails next time.

Needed for #TNTP-3709
diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml
@@ -40,20 +40,24 @@ jobs:
         with:
           tarantool-version: '${{ matrix.tarantool-version }}'
 
-      - name: Static code check
-        uses: ./.github/actions/static-code-check
+      # - name: Static code check
+      #   uses: ./.github/actions/static-code-check
 
-      - name: Unit tests
-        run: mage unitfull
+      # - name: Unit tests
+      #   run: mage unitfull
 
       # This server starts and listen on 8084 port that is used for tests.
       - name: Stop Mono server
-        run: sudo systemctl kill mono-xsp4 || true
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          sudo systemctl kill mono-xsp4 || echo "failed to kill mono-xsp4 (exit status $?)"
 
       - name: Integration tests
         env:
           TT_ENABLE_COREDUMP_TESTS: '1'
-        run: mage integrationfull
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          mage integrationfull
 
   full-ci-ce-linux-arm64:
     if: false
@@ -132,20 +136,24 @@ jobs:
           sdk-version: '${{ matrix.sdk-version }}'
           sdk-download-token: '${{ secrets.SDK_DOWNLOAD_TOKEN }}'
 
-      - name: Static code check
-        uses: ./.github/actions/static-code-check
+      # - name: Static code check
+      #   uses: ./.github/actions/static-code-check
 
-      - name: Unit tests
-        run: mage unitfull
+      # - name: Unit tests
+      #   run: mage unitfull
 
       # This server starts and listen on 8084 port that is used for tests.
       - name: Stop Mono server
-        run: sudo systemctl kill mono-xsp4 || true
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          sudo systemctl kill mono-xsp4 || echo "failed to kill mono-xsp4 (exit status $?)"
 
       - name: Integration tests
         env:
           TT_ENABLE_COREDUMP_TESTS: '1'
-        run: mage integrationfull
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          mage integrationfull
 
   full-ci-macOS:
     if: false
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -48,18 +48,22 @@ jobs:
         with:
           tarantool-version: '${{ matrix.tarantool-version }}'
 
-      - name: Static code check
-        uses: ./.github/actions/static-code-check
+      # - name: Static code check
+      #   uses: ./.github/actions/static-code-check
 
-      - name: Unit tests
-        run: mage unit
+      # - name: Unit tests
+      #   run: mage unit
 
       # This server starts and listen on 8084 port that is used for tests.
       - name: Stop Mono server
-        run: sudo systemctl kill mono-xsp4 || true
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          sudo systemctl kill mono-xsp4 || echo "failed to kill mono-xsp4 (exit status $?)"
 
       - name: Integration tests
-        run: mage integration
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          mage integration
 
   tests-ce-linux-arm64:
     if: false
@@ -138,18 +142,22 @@ jobs:
           sdk-version: '${{ matrix.sdk-version }}'
           sdk-download-token: '${{ secrets.SDK_DOWNLOAD_TOKEN }}'
 
-      - name: Static code check
-        uses: ./.github/actions/static-code-check
+      # - name: Static code check
+      #   uses: ./.github/actions/static-code-check
 
-      - name: Unit tests
-        run: mage unit
+      # - name: Unit tests
+      #   run: mage unit
 
       # This server starts and listen on 8084 port that is used for tests.
       - name: Stop Mono server
-        run: sudo systemctl kill mono-xsp4 || true
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          sudo systemctl kill mono-xsp4 || echo "failed to kill mono-xsp4 (exit status $?)"
 
       - name: Integration tests
-        run: mage integration
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          mage integration
 
   tests-mac-os-ce:
     if: false
diff --git a/cli/replicaset/cartridge.go b/cli/replicaset/cartridge.go
@@ -236,13 +236,17 @@ func (c *CartridgeApplication) discovery() (Replicasets, error) {
 		Orchestrator: OrchestratorCartridge,
 	}
 
+	newTopologies := make(map[string]cartridgeTopology, len(c.runningCtx.Instances))
+	evaledInstances := make([]string, 0, len(c.runningCtx.Instances))
 	var topology cartridgeTopology
 	err := EvalForeachAlive(c.runningCtx.Instances, InstanceEvalFunc(
 		func(inst running.InstanceCtx, evaler connector.Evaler) (bool, error) {
 			newTopology, err := getCartridgeTopology(evaler)
 			if err != nil {
 				return true, err
 			}
+			evaledInstances = append(evaledInstances, inst.InstName)
+			newTopologies[inst.InstName] = newTopology
 			if topology.IsBootstrapped {
 				if newTopology.IsBootstrapped {
 					topology = newTopology
@@ -252,13 +256,38 @@ func (c *CartridgeApplication) discovery() (Replicasets, error) {
 			}
 
 			// Stop if we already found a valid topology.
-			return topology.IsBootstrapped && !topology.IsCritical, nil
+			// return topology.IsBootstrapped && !topology.IsCritical, nil
+			return false, nil
 		},
 	))
 	if err != nil {
 		return replicasets, fmt.Errorf("failed to get topology: %w", err)
 	}
 
+	for _, instName := range evaledInstances {
+		newTopology := newTopologies[instName]
+		if topology.IsBootstrapped {
+			if newTopology.IsBootstrapped {
+				topology = newTopology
+			}
+		} else {
+			topology = newTopology
+		}
+	}
+
+	for _, instName := range evaledInstances {
+		t := newTopologies[instName]
+		fmt.Printf("CartridgeApplication.discovery: %s: topology:\n", instName)
+		fmt.Printf("  IsBootstrapped=%v\n", t.IsBootstrapped)
+		fmt.Printf("  IsCritical=%v\n", t.IsCritical)
+		for i, rs := range t.Replicasets {
+			fmt.Printf("  [%d]: Alias=%s UUID=%s LeaderUUID=%s\n", i, rs.Alias, rs.UUID, rs.LeaderUUID)
+			for ii, inst := range rs.Instances {
+				fmt.Printf("    [%d]: Alias=%s UUID=%s URI=%s\n", ii, inst.Alias, inst.UUID, inst.URI)
+			}
+		}
+	}
+
 	replicasets = getCartridgeReplicasets(topology)
 	err = EvalForeachAlive(c.runningCtx.Instances, InstanceEvalFunc(
 		func(ictx running.InstanceCtx, evaler connector.Evaler) (bool, error) {
@@ -498,6 +527,10 @@ func (c *CartridgeApplication) bootstrapInstance(instanceName, replicasetName st
 		UUID:        &replicasetUUID,
 		JoinServers: joinOpts,
 	}}
+	fmt.Println("bootstrapInstance: opts:")
+	for i, opt := range opts {
+		fmt.Printf("[%d]: %+v\n", i, opt)
+	}
 	return cartridgeEditReplicasets(evaler, opts, timeout)
 }
 
@@ -608,6 +641,7 @@ func updateCartridgeReplicasets(evaler connector.Evaler, discovered Replicasets,
 			opts.UUID = &uuid
 		}
 		var err error
+
 		opts.JoinServers, err = getCartridgeJoinServersOpts(instancesCfg,
 			rcfg.Instances, instanceUUID)
 		if err != nil {
@@ -625,7 +659,47 @@ func updateCartridgeReplicasets(evaler connector.Evaler, discovered Replicasets,
 		editOpts = append(editOpts, opts)
 	}
 
-	return cartridgeEditReplicasets(evaler, editOpts, timeout)
+	// return cartridgeEditReplicasets(evaler, editOpts, timeout)
+	err := cartridgeEditReplicasets(evaler, editOpts, timeout)
+
+	var dump []string = make([]string, 0, 40)
+	dump = append(dump, "updateCartridgeReplicasets: replicasetCfg:")
+	for k, v := range replicasetCfg {
+		dump = append(dump, fmt.Sprintf("  %s: %+v", k, v))
+	}
+	dump = append(dump, "updateCartridgeReplicasets: instancesCfg:")
+	for k, v := range instancesCfg {
+		dump = append(dump, fmt.Sprintf("  %s: %+v", k, v))
+	}
+	dump = append(dump, "updateCartridgeReplicasets: discovered:")
+	dump = append(dump, fmt.Sprintf("  State=%s", discovered.State))
+	dump = append(dump, fmt.Sprintf("  Orchestrator=%s", discovered.Orchestrator))
+	for i, rs := range discovered.Replicasets {
+		dump = append(dump, fmt.Sprintf("  [%d]: %+v", i, rs))
+	}
+
+	dump = append(dump, "updateCartridgeReplicasets: setup UUID maps")
+	for _, replicaset := range discovered.Replicasets {
+		dump = append(dump, fmt.Sprintf("  %s: %s", replicaset.Alias, replicaset.UUID))
+		for _, instance := range replicaset.Instances {
+			dump = append(dump, fmt.Sprintf("    %s: %s", instance.Alias, instance.UUID))
+		}
+	}
+	dump = append(dump, "updateCartridgeReplicasets: replicasetUUID:")
+	for k, v := range replicasetUUID {
+		dump = append(dump, fmt.Sprintf("  %s: %s", k, v))
+	}
+	dump = append(dump, "updateCartridgeReplicasets: instanceUUID:")
+	for k, v := range instanceUUID {
+		dump = append(dump, fmt.Sprintf("  %s: %s", k, v))
+	}
+	dump = append(dump, "updateCartridgeReplicasets: editOpts:")
+	for i, opt := range editOpts {
+		dump = append(dump, fmt.Sprintf("[%d]: %+v", i, opt))
+	}
+	fmt.Println(strings.Join(dump, "\n"))
+
+	return err
 }
 
 // Expel expels an instance from a Cartridge replicasets.
diff --git a/cli/replicaset/lua/cartridge/edit_replicasets_body.lua b/cli/replicaset/lua/cartridge/edit_replicasets_body.lua
@@ -5,4 +5,4 @@ local res, err = cartridge.admin_edit_topology({
     replicasets = replicasets
 })
 
-assert(res, tostring(err))
+assert(res, string.format("err:%s replicasets:%s", tostring(err), tostring(replicasets)))
diff --git a/magefile.go b/magefile.go
@@ -420,16 +420,18 @@ func (Unit) Coverage() error {
 func Integration() error {
 	fmt.Println("Running integration tests...")
 
+	test_name := "test_replicaset_bootstrap_cartridge_app_second_bootstrap"
 	return sh.RunV(pythonExecutableName, "-m", "pytest", "-m", "not slow and not slow_ee "+
-		"and not notarantool", "test/integration")
+		"and not notarantool", "-k", test_name, "--maxfail=1", "test/integration")
 }
 
 // Run full set of integration tests.
 func IntegrationFull() error {
 	fmt.Println("Running all integration tests...")
 
+	test_name := "test_replicaset_bootstrap_cartridge_app_second_bootstrap"
 	return sh.RunV(pythonExecutableName, "-m", "pytest", "-m", "not slow_ee and not notarantool",
-		"test/integration")
+		"-k", test_name, "--maxfail=1", "test/integration")
 }
 
 // Run full set of integration tests, excluding docker tests.
diff --git a/test/cartridge_helper.py b/test/cartridge_helper.py
@@ -175,6 +175,10 @@ def create(self):
         # Set replicasets config.
         with open(os.path.join(self.workdir, cartridge_name, "replicasets.yml"), "w") as f:
             f.write(yaml.dump(self.replicasets_cfg))
+        with open(os.path.join(self.workdir, cartridge_name, "instances.yml"), "r") as f:
+            print(f"CartridgeApp.create: instances.yml:\n{f.read()}")
+        with open(os.path.join(self.workdir, cartridge_name, "replicasets.yml"), "r") as f:
+            print(f"CartridgeApp.create: replicasets.yml:\n{f.read()}")
 
     def build(self):
         cmd = [self.tt_cmd, "build", cartridge_name]
@@ -193,6 +197,7 @@ def start(self, bootstrap_vshard=True):
         # Wait for the full start of the cartridge.
         for inst in self.instances:
             wait_inst_start(self.workdir, inst)
+        time.sleep(1)
 
         # Bootstrap.
         self.bootstrap(bootstrap_vshard=bootstrap_vshard)
diff --git a/test/integration/replicaset/test_replicaset_bootstrap.py b/test/integration/replicaset/test_replicaset_bootstrap.py
@@ -128,8 +128,10 @@ def test_bootstrap_app_replicaset_specified(tt_cmd, tmpdir_with_cfg):
 
 
 @pytest.mark.skipif(tarantool_major_version > 2, reason="skip cartridge test for Tarantool > 2")
+@pytest.mark.parametrize("attempt", range(100))
 @pytest.mark.parametrize("flag", [None, "--cartridge"])
-def test_replicaset_bootstrap_cartridge_app_second_bootstrap(tt_cmd, cartridge_app, flag):
+def test_replicaset_bootstrap_cartridge_app_second_bootstrap(tt_cmd, cartridge_app, flag, attempt):
+    print(f"Attempt #{attempt}")
     # Change the config.
     replicasets_cfg = {
         "router": {
@@ -159,8 +161,14 @@ def test_replicaset_bootstrap_cartridge_app_second_bootstrap(tt_cmd, cartridge_a
             "vshard_group": "default",
         },
     }
+    with open(os.path.join(cartridge_app.workdir, cartridge_name, "instances.yml"), "r") as f:
+        print(f"test: instances.yml:\n{f.read()}")
+    with open(os.path.join(cartridge_app.workdir, cartridge_name, "replicasets.yml"), "r") as f:
+        print(f"test: replicasets.yml #1:\n{f.read()}")
     with open(os.path.join(cartridge_app.workdir, cartridge_name, "replicasets.yml"), "w") as f:
         f.write(yaml.dump(replicasets_cfg))
+    with open(os.path.join(cartridge_app.workdir, cartridge_name, "replicasets.yml"), "r") as f:
+        print(f"test: replicasets.yml #2:\n{f.read()}")
 
     # Run bootstrap after initial bootstrap again.
     cmd = [tt_cmd, "rs", "bootstrap"]
diff --git a/test/utils.py b/test/utils.py
@@ -446,9 +446,11 @@ def find_ports(n=1, port=8000):
                 if s.connect_ex(("localhost", port)) == 0:
                     busy = True
                     break
+        print(f"find_ports({n}): is port {port} busy... {busy}")
         if not busy:
             ports.append(port)
         port += 1
+    print(f"find_ports({n}): {ports}")
     return ports
 
 
