ci: display port usage information

It might help to identify the reason of flaky test
`test_replicaset_bootstrap_cartridge_app_second_bootstrap`
when/if it fails next time.

Needed for #TNTP-3709

Author: elhimov

.github/workflows/full-ci.yml

@@ -40,20 +40,27 @@ jobs:
         with:
           tarantool-version: '${{ matrix.tarantool-version }}'
 
-      - name: Static code check
-        uses: ./.github/actions/static-code-check
+      # - name: Static code check
+      #   uses: ./.github/actions/static-code-check
 
-      - name: Unit tests
-        run: mage unitfull
+      # - name: Unit tests
+      #   run: mage unitfull
 
       # This server starts and listen on 8084 port that is used for tests.
       - name: Stop Mono server
-        run: sudo systemctl kill mono-xsp4 || true
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          sudo netstat -tupln
+          sudo systemctl kill mono-xsp4 || echo "failed to kill mono-xsp4 (exit status $?)"
 
       - name: Integration tests
         env:
           TT_ENABLE_COREDUMP_TESTS: '1'
-        run: mage integrationfull
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          sudo netstat -tupln
+          while mage integrationfull; do echo "test succeed, trying another attempt..."; done
+        # mage integrationfull || { sudo netstat -tupln; exit 1; }
 
   full-ci-ce-linux-arm64:
     if: false
@@ -132,20 +139,27 @@ jobs:
           sdk-version: '${{ matrix.sdk-version }}'
           sdk-download-token: '${{ secrets.SDK_DOWNLOAD_TOKEN }}'
 
-      - name: Static code check
-        uses: ./.github/actions/static-code-check
+      # - name: Static code check
+      #   uses: ./.github/actions/static-code-check
 
-      - name: Unit tests
-        run: mage unitfull
+      # - name: Unit tests
+      #   run: mage unitfull
 
       # This server starts and listen on 8084 port that is used for tests.
       - name: Stop Mono server
-        run: sudo systemctl kill mono-xsp4 || true
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          sudo netstat -tupln
+          sudo systemctl kill mono-xsp4 || echo "failed to kill mono-xsp4 (exit status $?)"
 
       - name: Integration tests
         env:
           TT_ENABLE_COREDUMP_TESTS: '1'
-        run: mage integrationfull
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          sudo netstat -tupln
+          while mage integrationfull; do echo "test succeed, trying another attempt..."; done
+        # mage integrationfull || { sudo netstat -tupln; exit 1; }
 
   full-ci-macOS:
     if: false

.github/workflows/tests.yml

@@ -48,18 +48,25 @@ jobs:
         with:
           tarantool-version: '${{ matrix.tarantool-version }}'
 
-      - name: Static code check
-        uses: ./.github/actions/static-code-check
+      # - name: Static code check
+      #   uses: ./.github/actions/static-code-check
 
-      - name: Unit tests
-        run: mage unit
+      # - name: Unit tests
+      #   run: mage unit
 
       # This server starts and listen on 8084 port that is used for tests.
       - name: Stop Mono server
-        run: sudo systemctl kill mono-xsp4 || true
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          sudo netstat -tupln
+          sudo systemctl kill mono-xsp4 || echo "failed to kill mono-xsp4 (exit status $?)"
 
       - name: Integration tests
-        run: mage integration
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          sudo netstat -tupln
+          while mage integration; do echo "test succeed, trying another attempt..."; done
+        # mage integration || { sudo netstat -tupln; exit 1; }
 
   tests-ce-linux-arm64:
     if: false
@@ -138,18 +145,25 @@ jobs:
           sdk-version: '${{ matrix.sdk-version }}'
           sdk-download-token: '${{ secrets.SDK_DOWNLOAD_TOKEN }}'
 
-      - name: Static code check
-        uses: ./.github/actions/static-code-check
+      # - name: Static code check
+      #   uses: ./.github/actions/static-code-check
 
-      - name: Unit tests
-        run: mage unit
+      # - name: Unit tests
+      #   run: mage unit
 
       # This server starts and listen on 8084 port that is used for tests.
       - name: Stop Mono server
-        run: sudo systemctl kill mono-xsp4 || true
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          sudo netstat -tupln
+          sudo systemctl kill mono-xsp4 || echo "failed to kill mono-xsp4 (exit status $?)"
 
       - name: Integration tests
-        run: mage integration
+        # netstat is used to help find the cause of TNTP-3709, remove it as soon as it's found.
+        run: |
+          sudo netstat -tupln
+          while mage integration; do echo "test succeed, trying another attempt..."; done
+        # mage integration || { sudo netstat -tupln; exit 1; }
 
   tests-mac-os-ce:
     if: false

cli/replicaset/cartridge.go

@@ -498,6 +498,10 @@ func (c *CartridgeApplication) bootstrapInstance(instanceName, replicasetName st
 		UUID:        &replicasetUUID,
 		JoinServers: joinOpts,
 	}}
+	fmt.Println("bootstrapInstance: opts:")
+	for i, opt := range opts {
+		fmt.Printf("[%d]: %+v\n", i, opt)
+	}
 	return cartridgeEditReplicasets(evaler, opts, timeout)
 }
 
@@ -608,6 +612,7 @@ func updateCartridgeReplicasets(evaler connector.Evaler, discovered Replicasets,
 			opts.UUID = &uuid
 		}
 		var err error
+
 		opts.JoinServers, err = getCartridgeJoinServersOpts(instancesCfg,
 			rcfg.Instances, instanceUUID)
 		if err != nil {
@@ -625,7 +630,47 @@ func updateCartridgeReplicasets(evaler connector.Evaler, discovered Replicasets,
 		editOpts = append(editOpts, opts)
 	}
 
-	return cartridgeEditReplicasets(evaler, editOpts, timeout)
+	// return cartridgeEditReplicasets(evaler, editOpts, timeout)
+	err := cartridgeEditReplicasets(evaler, editOpts, timeout)
+
+	var dump []string = make([]string, 0, 40)
+	dump = append(dump, "updateCartridgeReplicasets: replicasetCfg:")
+	for k, v := range replicasetCfg {
+		dump = append(dump, fmt.Sprintf("  %s: %+v", k, v))
+	}
+	dump = append(dump, "updateCartridgeReplicasets: instancesCfg:")
+	for k, v := range instancesCfg {
+		dump = append(dump, fmt.Sprintf("  %s: %+v", k, v))
+	}
+	dump = append(dump, "updateCartridgeReplicasets: discovered:")
+	dump = append(dump, fmt.Sprintf("  State=%s", discovered.State))
+	dump = append(dump, fmt.Sprintf("  Orchestrator=%s", discovered.Orchestrator))
+	for i, rs := range discovered.Replicasets {
+		dump = append(dump, fmt.Sprintf("  [%d]: %+v", i, rs))
+	}
+
+	dump = append(dump, "updateCartridgeReplicasets: setup UUID maps")
+	for _, replicaset := range discovered.Replicasets {
+		dump = append(dump, fmt.Sprintf("  %s: %s", replicaset.Alias, replicaset.UUID))
+		for _, instance := range replicaset.Instances {
+			dump = append(dump, fmt.Sprintf("    %s: %s", instance.Alias, instance.UUID))
+		}
+	}
+	dump = append(dump, "updateCartridgeReplicasets: replicasetUUID:")
+	for k, v := range replicasetUUID {
+		dump = append(dump, fmt.Sprintf("  %s: %s", k, v))
+	}
+	dump = append(dump, "updateCartridgeReplicasets: instanceUUID:")
+	for k, v := range instanceUUID {
+		dump = append(dump, fmt.Sprintf("  %s: %s", k, v))
+	}
+	dump = append(dump, "updateCartridgeReplicasets: editOpts:")
+	for i, opt := range editOpts {
+		dump = append(dump, fmt.Sprintf("[%d]: %+v", i, opt))
+	}
+	fmt.Println(strings.Join(dump, "\n"))
+
+	return err
 }
 
 // Expel expels an instance from a Cartridge replicasets.

cli/replicaset/lua/cartridge/edit_replicasets_body.lua

@@ -5,4 +5,4 @@ local res, err = cartridge.admin_edit_topology({
     replicasets = replicasets
 })
 
-assert(res, tostring(err))
+assert(res, string.format("err:%s replicasets:%s", tostring(err), tostring(replicasets)))

magefile.go

@@ -420,16 +420,18 @@ func (Unit) Coverage() error {
 func Integration() error {
 	fmt.Println("Running integration tests...")
 
+	test_name := "test_replicaset_bootstrap_cartridge_app_second_bootstrap"
 	return sh.RunV(pythonExecutableName, "-m", "pytest", "-m", "not slow and not slow_ee "+
-		"and not notarantool", "test/integration")
+		"and not notarantool", "-k", test_name, "--maxfail=1", "test/integration")
 }
 
 // Run full set of integration tests.
 func IntegrationFull() error {
 	fmt.Println("Running all integration tests...")
 
+	test_name := "test_replicaset_bootstrap_cartridge_app_second_bootstrap"
 	return sh.RunV(pythonExecutableName, "-m", "pytest", "-m", "not slow_ee and not notarantool",
-		"test/integration")
+		"-k", test_name, "--maxfail=1", "test/integration")
 }
 
 // Run full set of integration tests, excluding docker tests.

test/cartridge_helper.py

@@ -175,6 +175,10 @@ def create(self):
         # Set replicasets config.
         with open(os.path.join(self.workdir, cartridge_name, "replicasets.yml"), "w") as f:
             f.write(yaml.dump(self.replicasets_cfg))
+        with open(os.path.join(self.workdir, cartridge_name, "instances.yml"), "r") as f:
+            print(f"CartridgeApp.create: instances.yml:\n{f.read()}")
+        with open(os.path.join(self.workdir, cartridge_name, "replicasets.yml"), "r") as f:
+            print(f"CartridgeApp.create: replicasets.yml:\n{f.read()}")
 
     def build(self):
         cmd = [self.tt_cmd, "build", cartridge_name]

test/integration/replicaset/test_replicaset_bootstrap.py

@@ -128,8 +128,10 @@ def test_bootstrap_app_replicaset_specified(tt_cmd, tmpdir_with_cfg):
 
 
 @pytest.mark.skipif(tarantool_major_version > 2, reason="skip cartridge test for Tarantool > 2")
+@pytest.mark.parametrize("attempt", range(100))
 @pytest.mark.parametrize("flag", [None, "--cartridge"])
-def test_replicaset_bootstrap_cartridge_app_second_bootstrap(tt_cmd, cartridge_app, flag):
+def test_replicaset_bootstrap_cartridge_app_second_bootstrap(tt_cmd, cartridge_app, flag, attempt):
+    print(f"Attempt #{attempt}")
     # Change the config.
     replicasets_cfg = {
         "router": {
@@ -159,8 +161,14 @@ def test_replicaset_bootstrap_cartridge_app_second_bootstrap(tt_cmd, cartridge_a
             "vshard_group": "default",
         },
     }
+    with open(os.path.join(cartridge_app.workdir, cartridge_name, "instances.yml"), "r") as f:
+        print(f"test: instances.yml:\n{f.read()}")
+    with open(os.path.join(cartridge_app.workdir, cartridge_name, "replicasets.yml"), "r") as f:
+        print(f"test: replicasets.yml #1:\n{f.read()}")
     with open(os.path.join(cartridge_app.workdir, cartridge_name, "replicasets.yml"), "w") as f:
         f.write(yaml.dump(replicasets_cfg))
+    with open(os.path.join(cartridge_app.workdir, cartridge_name, "replicasets.yml"), "r") as f:
+        print(f"test: replicasets.yml #2:\n{f.read()}")
 
     # Run bootstrap after initial bootstrap again.
     cmd = [tt_cmd, "rs", "bootstrap"]

test/utils.py

@@ -446,9 +446,11 @@ def find_ports(n=1, port=8000):
                 if s.connect_ex(("localhost", port)) == 0:
                     busy = True
                     break
+        print(f"find_ports({n}): is port {port} busy... {busy}")
         if not busy:
             ports.append(port)
         port += 1
+    print(f"find_ports({n}): {ports}")
     return ports