Fix addr conflict.

infiniflow · Dec 18, 2024 · b5515c7 · b5515c7
1 parent d347170
commit b5515c7
Show file tree

Hide file tree

Showing 8 changed files with 92 additions and 266 deletions.
diff --git a/.github/workflows/slow_test.yml b/.github/workflows/slow_test.yml
@@ -60,8 +60,6 @@ jobs:
           echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> $GITHUB_ENV
           echo "CPUS=${CPUS}" >> $GITHUB_ENV
           TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
-          INF_DIRECTORY=$PWD
-          echo "${INF_DIRECTORY}" | sudo tee .tester_env
           sudo docker pull infiniflow/infinity_builder:centos7_clang18
           sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run --net=host --privileged --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot -v /var/run/docker.sock:/var/run/docker.sock --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18
 
@@ -113,7 +111,7 @@ jobs:
           # Fix sanitizer: https://github.com/ClickHouse/ClickHouse/issues/64086
           old_value=$(sudo sysctl -n vm.mmap_rnd_bits)
           sudo sysctl -w vm.mmap_rnd_bits=28
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=${INF_DIRECTORY} --minio_port=9005 --minio_console_port=9006"
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity"
           sudo sysctl -w vm.mmap_rnd_bits=$old_value
 
       - name: Collect thread sanitizer output in cluster test

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -54,10 +54,21 @@ jobs:
           echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> $GITHUB_ENV
           echo "CPUS=${CPUS}" >> $GITHUB_ENV
           TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
-          INF_DIRECTORY=$PWD
-          echo "${INF_DIRECTORY}" | sudo tee .tester_env
           sudo docker pull infiniflow/infinity_builder:centos7_clang18
-          sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run --net=host --privileged --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot -v /var/run/docker.sock:/var/run/docker.sock --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18
+          sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot -v /var/run/docker.sock:/var/run/docker.sock --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18
+
+      - name: Start minio container
+        if: ${{ !cancelled() && !failure() }}
+        run: |
+          MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
+          MINIO_DIR=~/minio_data_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
+          echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> $GITHUB_ENV
+          echo "MINIO_DIR=${MINIO_DIR}" >> $GITHUB_ENV
+          sudo docker rm -f -v ${MINIO_CONTAINER} && sudo rm -fr ${MINIO_DIR} && sudo mkdir ${MINIO_DIR} && sudo docker run -d --net=container:${BUILDER_CONTAINER} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin"  -v ${MINIO_DIR}:/data    quay.io/minio/minio server /data --console-address ":9006" --address ":9005" && sleep 5s
+          if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then
+              echo "Minio container is not running"
+              exit 1
+          fi
 
       - name: Build debug version
         if: ${{ !cancelled() && !failure() }}
@@ -71,7 +82,7 @@ jobs:
         if: ${{ !cancelled() && !failure() }}
         id: run_cluster_test
         run: |
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=${INF_DIRECTORY} --minio_port=9005 --minio_console_port=9006"
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity"
 
       - name: Collect cluster test output
         if: ${{ !cancelled() }}
@@ -96,19 +107,6 @@ jobs:
           failure="${{ steps.run_restart_test.outcome == 'failure'}}"
           sudo python3 scripts/collect_restart_log.py --executable_path=cmake-build-debug/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
-      - name: Start minio container
-        if: ${{ !cancelled() && !failure() }}
-        run: |
-          MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
-          MINIO_DIR=~/minio_data_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
-          echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> $GITHUB_ENV
-          echo "MINIO_DIR=${MINIO_DIR}" >> $GITHUB_ENV
-          sudo docker rm -f -v ${MINIO_CONTAINER} && sudo rm -fr ${MINIO_DIR} && sudo mkdir ${MINIO_DIR} && sudo docker run -d --net=container:${BUILDER_CONTAINER} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin"  -v ${MINIO_DIR}:/data    quay.io/minio/minio server /data --console-address ":9006" --address ":9005" && sleep 5s
-          if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then
-              echo "Minio container is not running"
-              exit 1
-          fi
-
       - name: Start infinity debug version with minio
         if: ${{ !cancelled() && !failure() }}
         run: |
@@ -235,6 +233,18 @@ jobs:
           TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
           sudo docker pull infiniflow/infinity_builder:centos7_clang18
           sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18
+      - name: Start minio container
+        if: ${{ !cancelled() && !failure() }}
+        run: |
+          MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
+          MINIO_DIR=~/minio_data_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
+          echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> $GITHUB_ENV
+          echo "MINIO_DIR=${MINIO_DIR}" >> $GITHUB_ENV
+          sudo docker rm -f -v ${MINIO_CONTAINER} && sudo rm -fr ${MINIO_DIR} && sudo mkdir ${MINIO_DIR} && sudo docker run -d --net=container:${BUILDER_CONTAINER} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin"  -v ${MINIO_DIR}:/data    quay.io/minio/minio server /data --console-address ":9006" --address ":9005" && sleep 5s
+          if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then
+              echo "Minio container is not running"
+              exit 1
+          fi
 
       - name: Build release version
         if: ${{ !cancelled() && !failure() }}
@@ -244,17 +254,17 @@ jobs:
         if: ${{ !cancelled() && !failure() }}
         run: sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk infinity-embedded-sdk && pip3 install . -v --config-settings=cmake.build-type='RelWithDebInfo' --config-settings=build-dir='cmake-build-release' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd python/infinity_sdk/ && pip3 install . -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.."
 
-    #   - name: Run cluster test
-    #     if: ${{ !cancelled() && !failure() }}
-    #     id: run_cluster_test
-    #     run: |
-    #       sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-release/src/infinity --infinity_dir=${INF_DIRECTORY} --minio_port=9005 --minio_console_port=9006"
-
-    #   - name: Collect cluster test output
-    #     if: ${{ !cancelled() }}
-    #     run: |
-    #       failure="${{ steps.run_cluster_test.outcome == 'failure'}}"
-    #       sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-release/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
+      - name: Run cluster test
+        if: ${{ !cancelled() && !failure() }}
+        id: run_cluster_test
+        run: |
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-release/src/infinity"
+
+      - name: Collect cluster test output
+        if: ${{ !cancelled() }}
+        run: |
+          failure="${{ steps.run_cluster_test.outcome == 'failure'}}"
+          sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-release/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
     
       - name: Prepare restart test data
         if: ${{ !cancelled() && !failure() }}
@@ -284,19 +294,6 @@ jobs:
           failure="${{ steps.run_pysdk_local_infinity_test.outcome == 'failure'}}"
           sudo python3 scripts/collect_embedded_log.py --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure} --log_path=/var/infinity/log/infinity.log
 
-      - name: Start minio container
-        if: ${{ !cancelled() && !failure() }}
-        run: |
-          MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
-          MINIO_DIR=~/minio_data_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
-          echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> $GITHUB_ENV
-          echo "MINIO_DIR=${MINIO_DIR}" >> $GITHUB_ENV
-          sudo docker rm -f -v ${MINIO_CONTAINER} && sudo rm -fr ${MINIO_DIR} && sudo mkdir ${MINIO_DIR} && sudo docker run -d --net=container:${BUILDER_CONTAINER} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin"  -v ${MINIO_DIR}:/data    quay.io/minio/minio server /data --console-address ":9006" --address ":9005" && sleep 5s
-          if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then
-              echo "Minio container is not running"
-              exit 1
-          fi
-
       - name: Start infinity release version with minio
         if: ${{ !cancelled() && !failure() }}
         run: |

diff --git a/python/test_cluster/clear_docker.py b/python/test_cluster/clear_docker.py
diff --git a/python/test_cluster/conftest.py b/python/test_cluster/conftest.py
@@ -1,6 +1,5 @@
 import pytest
 from infinity_cluster import InfinityCluster, MinioParams
-from docker_infinity_cluster import DockerInfinityCluster
 from mocked_infinity_cluster import MockInfinityCluster
 
 
@@ -10,28 +9,6 @@ def pytest_addoption(parser):
         action="store",
         default="cmake-build-debug/src/infinity",
     )
-    parser.addoption(
-        "--minio_dir",
-        action="store",
-        default="minio",
-    )
-    parser.addoption(
-        "--minio_port",
-        action="store",
-        default=9000,
-    )
-    parser.addoption(
-        "--minio_console_port",
-        action="store",
-        default=9001,
-    )
-    parser.addoption(
-        "--infinity_dir",
-        action="store",
-        required=True,
-        help="Path to infinity directory. For local test, $pwd is ok",
-    )
-    parser.addoption("--docker", action="store_true", default=False)
     parser.addoption(
         "--use_sudo",
         action="store_true",
@@ -40,56 +17,19 @@ def pytest_addoption(parser):
     )
 
 
-def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "docker: mark test to run only when --docker option is provided"
-    )
-
-
-def pytest_collection_modifyitems(config, items):
-    if config.getoption("--docker"):
-        return  # do not skip docker test
-    skip_docker = pytest.mark.skip(reason="need --docker option to run")
-    for item in items:
-        if "docker" in item.keywords:
-            print(f"skip {item.name}")
-            item.add_marker(skip_docker)
-
-
 def pytest_generate_tests(metafunc):
     infinity_path = metafunc.config.getoption("infinity_path")
-    minio_dir = metafunc.config.getoption("minio_dir")
-    minio_port = metafunc.config.getoption("minio_port")
-    minio_console_port = metafunc.config.getoption("minio_console_port")
-    minio_params = MinioParams(minio_dir, minio_port, minio_console_port)
 
-    infinity_dir = metafunc.config.getoption("infinity_dir")
     use_sudo = metafunc.config.getoption("use_sudo")
 
-    if len(infinity_dir) == 0:
-        # raise ValueError("Please provide a valid infinity_dir")
-        pass
-    print("infinity_dir: ", infinity_dir)
-    # print(metafunc.fixturenames)
-
     test_name = metafunc.function.__name__
-    if "docker_cluster" in metafunc.fixturenames:
-        # skip if docker is in option and the testcase is marked with docker
-        if not metafunc.config.getoption("--docker"):
-            return
-        docker_infinity_cluster = DockerInfinityCluster(
-            infinity_path, minio_params=minio_params, infinity_dir=infinity_dir
-        )
-        metafunc.parametrize("docker_cluster", [docker_infinity_cluster])
-    elif "cluster" in metafunc.fixturenames:
-        infinity_cluster = InfinityCluster(
-            infinity_path, minio_params=minio_params, test_name=test_name
-        )
+
+    if "cluster" in metafunc.fixturenames:
+        infinity_cluster = InfinityCluster(infinity_path, test_name=test_name)
         metafunc.parametrize("cluster", [infinity_cluster])
     elif "mock_cluster" in metafunc.fixturenames:
         mock_infinity_cluster = MockInfinityCluster(
             infinity_path,
-            minio_params=minio_params,
             test_name=test_name,
             use_sudo=use_sudo,
         )

diff --git a/python/test_cluster/infinity_cluster.py b/python/test_cluster/infinity_cluster.py
@@ -161,14 +161,13 @@ def load_config(self):
 
 class InfinityCluster:
     def __init__(
-        self, executable_path: str, *, minio_params: MinioParams, test_name: str = None
+        self, executable_path: str, *, test_name: str = None
     ):
         self.executable_path = executable_path
         self.runners: dict[str, InfinityRunner] = {}
         self.leader_runner: InfinityRunner | None = None
         self.leader_name = None
         self.logger_name = test_name
-        self.minio_params = minio_params
 
     def _log_filename(self):
         if self.logger_name is None:
@@ -190,7 +189,6 @@ def init_logger(self, logger_name: str):
 
     def __enter__(self):
         self.init_logger(self.logger_name)
-        self.add_minio(self.minio_params)
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
@@ -218,9 +216,6 @@ def clear_log(self):
         if os.path.exists(log_file):
             os.remove(log_file)
 
-        # if self.minio_container is not None:
-        #     self.minio_container.remove(force=True, v=True)
-
     def add_node(self, node_name: str, config_path: str, init=True):
         runner = InfinityRunner(
             node_name, self.executable_path, config_path, init, self.logger
@@ -242,33 +237,6 @@ def add_node(self, node_name: str, config_path: str, init=True):
             raise ValueError(f"Node {node_name} already exists in the cluster.")
         self.runners[node_name] = runner
 
-    def add_minio(self, minio_params: MinioParams):
-        minio_image_name = "quay.io/minio/minio"
-
-        minio_cmd = f'server /data --address ":{minio_params.minio_port}" --console-address ":{minio_params.minio_console_port}"'
-        docker_client = docker.from_env()
-        container_name = "minio_host"
-
-        try:
-            self.minio_container = docker_client.containers.get(container_name)
-            print(self.minio_container)
-            self.minio_container.start()
-            self.logger.debug(f"Minio container {container_name} already exists.")
-        except docker.errors.NotFound:
-            self.minio_container = docker_client.containers.run(
-                image=minio_image_name,
-                name=container_name,
-                detach=True,
-                environment=[
-                    "MINIO_ROOT_PASSWORD=minioadmin",
-                    "MINIO_ROOT_USER=minioadmin",
-                ],
-                volumes=[f"{minio_params.minio_dir}:/data"],
-                command=minio_cmd,
-                network="host",
-            )
-            self.logger.debug(f"Minio container {container_name} created.")
-
     def set_standalone(self, node_name: str):
         if self.leader_runner is not None and self.leader_name == node_name:
             self.leader_name = None

diff --git a/python/test_cluster/test_basic.py b/python/test_cluster/test_basic.py
@@ -67,33 +67,33 @@ def test_0(cluster: InfinityCluster):
 #         cluster.remove_node("node1")
 
 
-@pytest.mark.docker
-def test_docker(docker_cluster: DockerInfinityCluster):
-    cluster = docker_cluster
+# @pytest.mark.docker
+# def test_docker(docker_cluster: DockerInfinityCluster):
+#     cluster = docker_cluster
 
-    cluster.add_node("node1", "conf/leader.toml")
-    cluster.add_node("node2", "conf/follower.toml")
+#     cluster.add_node("node1", "conf/leader.toml")
+#     cluster.add_node("node2", "conf/follower.toml")
 
-    print("init nodes")
+#     print("init nodes")
 
-    cluster.set_leader("node1")
-    cluster.set_follower("node2")
+#     cluster.set_leader("node1")
+#     cluster.set_follower("node2")
 
-    time.sleep(1)
+#     time.sleep(1)
 
-    cluster.disconnect("node2")
-    time.sleep(0.1)
-    cluster.reconnect("node2")
+#     cluster.disconnect("node2")
+#     time.sleep(0.1)
+#     cluster.reconnect("node2")
 
-    res = cluster.client("node1").list_databases()
-    print(res.db_names)
+#     res = cluster.client("node1").list_databases()
+#     print(res.db_names)
 
-    time.sleep(1)
+#     time.sleep(1)
 
-    print("remove nodes")
+#     print("remove nodes")
 
-    cluster.clear()
+#     cluster.clear()
 
-# n1 admin, n2 admin, n2 connect n1 as follower-> failed, n2 connect n1 as learner-> failed
-# n1 leader, n2 follower, n3 connect n2 as follower-> failed, n3 connect n2 as learner-> failed
-#
+# # n1 admin, n2 admin, n2 connect n1 as follower-> failed, n2 connect n1 as learner-> failed
+# # n1 leader, n2 follower, n3 connect n2 as follower-> failed, n3 connect n2 as learner-> failed
+# #