Fix.

infiniflow · Oct 31, 2024 · 1013b7d · 1013b7d
1 parent 520372f
commit 1013b7d
Show file tree

Hide file tree

Showing 10 changed files with 151 additions and 53 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -63,28 +63,43 @@ jobs:
       - name: Install pysdk
         if: ${{ !cancelled() && !failure() }}
         run: sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk infinity-embedded-sdk && pip3 install . -v --config-settings=cmake.build-type='Debug' --config-settings=build-dir='cmake-build-debug' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd python/infinity_sdk/ && pip3 install . -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.."
+
+      - name: Create Test image
+        if: ${{ !cancelled() && !failure() }}
+        run: |
+            wget https://mirrors.edge.kernel.org/pub/linux/utils/net/iproute2/iproute2-6.9.0.tar.gz
+            sudo docker build -f scripts/Dockerfile_infinity_tester_centos7 -t infiniflow/infinity_tester:centos7_clang18 .
+            echo docker image ls
+
+      - name: Start tester container
+        if: ${{ !cancelled() && !failure() }}
+        run: |
+            TESTER_CONTAINER=infinity_tester_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
+            CPUS=${CPUS:-$(nproc)}
+            echo "TESTER_CONTAINER=${TESTER_CONTAINER}" >> $GITHUB_ENV
+            echo "CPUS=${CPUS}" >> $GITHUB_ENV
+            TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
+            sudo docker rm -f -v ${TESTER_CONTAINER} && sudo docker run --net=host --cap-add=NET_ADMIN -d --name ${TESTER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -e INFINITY_DIR=$PWD -v $PWD:/infinity -v /boot:/boot -v /var/run/docker.sock:/var/run/docker.sock --cpus ${CPUS} infiniflow/infinity_tester:centos7_clang18
     
-    #   - name: Prepare cluster test
-    #     if: ${{ !cancelled() && !failure() }}
-    #     run : |
-    #       sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/test_cluster/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn"
-    #     #   sudo docker exec ${BUILDER_CONTAINER} bash -c "yum install sudo iproute2 bridge-utils -y"
-    #     #   sudo apt install virtualenv -y
-    #     #   virtualenv -p python3 .venv
-    #     #   source .venv/bin/activate && pip3 install pytest && pip3 install -r python/test_cluster/requirements.txt
-
-    #   - name: Run cluster test
-    #     if: ${{ !cancelled() && !failure() }}
-    #     id: run_cluster_test
-    #     run: |
-    #       sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity"
-    #     #   source .venv/bin/activate && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --docker
+      - name: Run cluster test
+        if: ${{ !cancelled() && !failure() }}
+        id: run_cluster_test
+        run: |
+          sudo docker exec ${TESTER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=$INFINITY_DIR"
+          sudo docker exec ${TESTER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=$INFINITY_DIR --docker"
 
-    #   - name: Collect cluster test output
-    #     if: ${{ !cancelled() }}
-    #     run: |
-    #       failure="${{ steps.run_cluster_test.outcome == 'failure'}}"
-    #       sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-debug/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
+      - name: Collect cluster test output
+        if: ${{ !cancelled() }}
+        run: |
+          failure="${{ steps.run_cluster_test.outcome == 'failure'}}"
+          sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-debug/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
+
+      - name: Remove tester container
+        if: always()  # always run this step even if previous steps failed
+        run: |
+          if [ -n "${TESTER_CONTAINER}" ]; then
+            sudo docker rm -f -v ${TESTER_CONTAINER}
+          fi
     
       - name: Prepare restart test data
         if: ${{ !cancelled() && !failure() }}

diff --git a/.gitignore b/.gitignore
@@ -139,3 +139,6 @@ tmp*/
 dist/
 python/infinity_sdk/infinity_sdk.egg-info/
 minio/
+
+# not ignore Dockerfile
+!Dockerfile*
diff --git a/python/test_cluster/conftest.py b/python/test_cluster/conftest.py
@@ -1,3 +1,4 @@
+import os
 import pytest
 from infinity_cluster import InfinityCluster, MinioParams
 from docker_infinity_cluster import DockerInfinityCluster
@@ -37,25 +38,35 @@ def pytest_addoption(parser):
         action="store",
         default=9001,
     )
+    parser.addoption(
+        "--infinity_dir",
+        action="store",
+        default="",
+    )
 
 
 def pytest_generate_tests(metafunc):
     infinity_path = metafunc.config.getoption("infinity_path")
     minio_dir = metafunc.config.getoption("minio_dir")
     minio_port = metafunc.config.getoption("minio_port")
     minio_params = MinioParams(minio_dir, minio_port)
+
+    infinity_dir = metafunc.config.getoption("infinity_dir")
+    if len(infinity_dir) == 0:
+        infinity_dir = os.getcwd()
+    print("infinity_dir: ", infinity_dir)
     # print(metafunc.fixturenames)
 
     if metafunc.config.getoption("--docker"):
         print("Docker argument is provided")
         if "docker_cluster" in metafunc.fixturenames:
-            docker_infinity_cluster = DockerInfinityCluster(infinity_path, minio_params=minio_params)
+            docker_infinity_cluster = DockerInfinityCluster(infinity_path, minio_params=minio_params, infinity_dir=infinity_dir)
             metafunc.parametrize("docker_cluster", [docker_infinity_cluster])
     else:
         print("Docker argument is not provided")
         if "cluster" in metafunc.fixturenames:
-            infinity_cluster = InfinityCluster(infinity_path, minio_params=minio_params)
+            infinity_cluster = InfinityCluster(infinity_path, minio_params=minio_params, infinity_dir=infinity_dir)
             metafunc.parametrize("cluster", [infinity_cluster])
         elif "mock_cluster" in metafunc.fixturenames:
-            mock_infinity_cluster = MockInfinityCluster(infinity_path, minio_params=minio_params)
+            mock_infinity_cluster = MockInfinityCluster(infinity_path, minio_params=minio_params, infinity_dir=infinity_dir)
             metafunc.parametrize("mock_cluster", [mock_infinity_cluster])
diff --git a/python/test_cluster/docker_infinity_cluster.py b/python/test_cluster/docker_infinity_cluster.py
@@ -84,7 +84,7 @@ def init(self, config_path: str | None):
         run_cmds = " && ".join(
             [
                 "cd /infinity",
-                f"{self.executable_path} --config={self.config_path}",
+                f"{self.executable_path} --config={self.config_path} 2>&1",
             ]
         )
         print(run_cmds)
@@ -155,12 +155,9 @@ def __init__(
         executable_path: str,
         *,
         minio_params: MinioParams = None,
+        infinity_dir: str,
     ):
-        super().__init__(executable_path)
-
-        image_name = "infiniflow/infinity_builder:centos7_clang18"
         docker_client = docker.from_env()
-        self.image_name = image_name
 
         network_name = "infinity_network"
         try:
@@ -170,9 +167,15 @@ def __init__(
                 network_name,
                 driver="bridge",
             )
+        super().__init__(
+            executable_path, minio_params=minio_params, infinity_dir=infinity_dir
+        )
+
+        image_name = "infiniflow/infinity_builder:centos7_clang18"
+        self.image_name = image_name
 
         if minio_params is not None:
-            add = self.add_minio(minio_params, False)
+            add = self.add_minio(minio_params)
             if add:
                 self.network.connect(self.minio_container)
             info = docker_client.api.inspect_network(self.network.id)
@@ -181,6 +184,36 @@ def __init__(
             self.minio_ip = minio_ip
             self.minio_params = minio_params
 
+    def add_minio(self, minio_params: MinioParams):
+        minio_image_name = "quay.io/minio/minio"
+
+        minio_cmd = f'server /data --console-address ":{minio_params.minio_port}"'
+        docker_client = docker.from_env()
+        kargs = {}
+        container_name = "minio_docker"
+
+        try:
+            self.minio_container = docker_client.containers.get(container_name)
+        except docker.errors.NotFound:
+            self.minio_container = docker_client.containers.run(
+                image=minio_image_name,
+                name=container_name,
+                detach=True,
+                environment=[
+                    "MINIO_ROOT_PASSWORD=minioadmin",
+                    "MINIO_ROOT_USER=minioadmin",
+                ],
+                volumes=[f"{minio_params.minio_dir}:/data"],
+                command=minio_cmd,
+                **kargs,
+            )
+            self.network.connect(self.minio_container)
+        info = docker_client.api.inspect_network(self.network.id)
+        minio_ip = info["Containers"][self.minio_container.id]["IPv4Address"]
+        minio_ip = minio_ip.split("/")[0]
+        self.minio_ip = minio_ip
+        self.minio_params = minio_params
+
     def clear(self):
         super().clear()
         for runner in self.runners.values():
@@ -191,7 +224,8 @@ def add_node(self, node_name: str, config_path: str):
         if node_name in self.runners:
             raise ValueError(f"Node {node_name} already exists in the cluster.")
         container_name, cpus, tz = self.__init_docker_params()
-        pwd = os.getcwd()
+        pwd = self.infinity_dir
+        print(f"pwd: {pwd}")
         docker_client = docker.from_env()
 
         try:
@@ -203,7 +237,10 @@ def add_node(self, node_name: str, config_path: str):
                 name=container_name,
                 detach=True,
                 cpuset_cpus=f"0-{cpus - 1}",
-                volumes=[f"{pwd}:/infinity", "/boot:/boot"],
+                volumes=["/boot:/boot"],
+                mounts=[
+                    docker.types.Mount(target="/infinity", source=pwd, type="bind")
+                ],
                 environment=[f"TZ={tz}"],
             )
             added = True

diff --git a/python/test_cluster/infinity_cluster.py b/python/test_cluster/infinity_cluster.py
@@ -143,13 +143,13 @@ def load_config(self):
 
 
 class InfinityCluster:
-    def __init__(self, executable_path: str, *, minio_params: MinioParams = None):
+    def __init__(self, executable_path: str, *, minio_params: MinioParams, infinity_dir: str):
         self.executable_path = executable_path
         self.runners: dict[str, InfinityRunner] = {}
         self.leader_runner: InfinityRunner | None = None
+        self.infinity_dir = infinity_dir
 
-        if minio_params is not None:
-            self.add_minio(minio_params, True)
+        self.add_minio(minio_params)
 
     def clear(self):
         for runner in self.runners.values():
@@ -163,21 +163,15 @@ def add_node(self, node_name: str, config_path: str):
             raise ValueError(f"Node {node_name} already exists in the cluster.")
         self.runners[node_name] = runner
 
-    def add_minio(self, minio_params: MinioParams, host_net: bool) -> bool:
+    def add_minio(self, minio_params: MinioParams):
         minio_image_name = "quay.io/minio/minio"
 
         minio_cmd = f'server /data --console-address ":{minio_params.minio_port}"'
         docker_client = docker.from_env()
-        kargs = {}
-        if host_net:
-            kargs = {"network": "host"}
-            container_name = "minio_host"
-        else:
-            container_name = "minio_docker"
+        container_name = "minio_host"
 
         try:
             self.minio_container = docker_client.containers.get(container_name)
-            return False
         except docker.errors.NotFound:
             self.minio_container = docker_client.containers.run(
                 image=minio_image_name,
@@ -189,9 +183,8 @@ def add_minio(self, minio_params: MinioParams, host_net: bool) -> bool:
                 ],
                 volumes=[f"{minio_params.minio_dir}:/data"],
                 command=minio_cmd,
-                **kargs,
+                network="host",
             )
-            return True
 
     def init_standalone(self, node_name: str):
         if node_name not in self.runners:

diff --git a/python/test_cluster/mocked_infinity_cluster.py b/python/test_cluster/mocked_infinity_cluster.py
@@ -3,7 +3,12 @@
 import platform
 import subprocess
 import sys
-from infinity_cluster import InfinityRunner, InfinityCluster, MinioParams, convert_request_to_curl
+from infinity_cluster import (
+    InfinityRunner,
+    InfinityCluster,
+    MinioParams,
+    convert_request_to_curl,
+)
 import os
 
 current_dir = os.path.dirname(os.path.abspath(__file__))
@@ -12,6 +17,7 @@
     sys.path.insert(0, parent_dir)
 from infinity_http import http_network_util, infinity_http
 
+
 class mocked_http_network(http_network_util):
     def __init__(self, ns_name: str, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -67,8 +73,14 @@ def peer_uri(self):
 
 
 class MockInfinityCluster(InfinityCluster):
-    def __init__(self, executable_path: str, *, minio_params: MinioParams = None):
-        super().__init__(executable_path, minio_params=minio_params)
+    def __init__(
+        self,
+        executable_path: str,
+        *,
+        minio_params: MinioParams = None,
+        infinity_dir: str,
+    ):
+        super().__init__(executable_path, minio_params=minio_params, infinity_dir=infinity_dir)
         self.ns_prefix = "ns"
         self.bridge_name = "br0"
         self.mock_ip_prefix = "17.0.0."

diff --git a/scripts/Dockerfile_infinity_builder_centos7 b/scripts/Dockerfile_infinity_builder_centos7
@@ -240,9 +240,4 @@ RUN git clone --single-branch --depth=1 https://github.com/VectorCamp/vectorscan
     && make -j 12 install \
     && rm -rf ../../vectorscan
 
-# Install Docker
-# https://docs.docker.com/engine/install/centos/#install-using-the-convenience-script
-RUN cd /root && curl -fsSL https://get.docker.com -o get-docker.sh \
-    && sh get-docker.sh
-
 ENTRYPOINT [ "bash", "-c", "while true; do sleep 60; done"]
diff --git a/scripts/Dockerfile_infinity_tester_centos7 b/scripts/Dockerfile_infinity_tester_centos7
@@ -0,0 +1,24 @@
+FROM infiniflow/infinity_builder:centos7_clang18
+
+# Install Docker
+# https://docs.docker.com/engine/install/centos/#install-using-the-convenience-script
+RUN cd /root && curl -fsSL https://get.docker.com -o get-docker.sh \
+    && sh get-docker.sh
+
+# Install iproute2 manually
+RUN --mount=type=bind,source=iproute2-6.9.0.tar.gz,target=/root/iproute2-6.9.0.tar.gz \
+    cd /root \
+    && tar -zxf iproute2-6.9.0.tar.gz && cd iproute2-6.9.0 \
+    && ./configure --prefix=/usr/local/iproute2 \
+    && make && make install \
+    && ip -V
+
+# test requirements
+RUN --mount=type=bind,source=python/test_cluster/requirements.txt,target=/root/requirements1.txt \
+    cd /root \
+    && /usr/local/bin/python3.10 -m venv /usr/local/venv310 && source /usr/local/venv310/bin/activate \
+    && pip3 install -r requirements1.txt
+
+RUN yum install -y sudo bridge-utils
+
+ENTRYPOINT [ "bash", "-c", "while true; do sleep 60; done"]
diff --git a/scripts/download_deps_infinity_builder_centos7.sh b/scripts/download_deps_infinity_builder_centos7.sh
@@ -34,7 +34,8 @@ https://www.python.org/ftp/python/3.10.15/Python-3.10.15.tar.xz
 https://www.python.org/ftp/python/3.11.10/Python-3.11.10.tar.xz
 https://www.python.org/ftp/python/3.12.7/Python-3.12.7.tar.xz
 https://www.python.org/ftp/python/3.13.0/Python-3.13.0.tar.xz
-https://github.com/risinglightdb/sqllogictest-rs/releases/download/v0.20.2/sqllogictest-bin-v0.20.2-x86_64-unknown-linux-musl.tar.gz"
+https://github.com/risinglightdb/sqllogictest-rs/releases/download/v0.20.2/sqllogictest-bin-v0.20.2-x86_64-unknown-linux-musl.tar.gz
+https://mirrors.edge.kernel.org/pub/linux/utils/net/iproute2/iproute2-6.9.0.tar.gz"
 
 SAVEIFS=$IFS   # Save current IFS (Internal Field Separator)
 IFS=$'\n'      # Change IFS to newline char

diff --git a/tools/run_cluster_test.py b/tools/run_cluster_test.py
@@ -17,10 +17,16 @@
         action="store_true",
         default=False,
     )
+    parser.add_argument(
+        "--infinity_dir",
+        type=str,
+        required=True,
+    )
 
     args = parser.parse_args()
     infinity_path = args.infinity_path
     docker = args.docker
+    infinity_dir = args.infinity_dir
 
     current_path = os.getcwd()
     python_test_dir = current_path + "/python"
@@ -35,6 +41,7 @@
         "-s",
         "-m",
         "not slow",
+        f"--infinity_dir={infinity_dir}",
     ]
     if docker:
         cmd.append("--docker")