Refactor code #12955
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: tests | |
on: | |
push: | |
branches: | |
- 'main' | |
- 'clang-20' | |
- '*.*.*' | |
paths-ignore: | |
- 'docs/**' | |
- '*.md' | |
pull_request: | |
types: [ opened, synchronize, reopened, labeled ] | |
paths-ignore: | |
- 'docs/**' | |
- '*.md' | |
# https://docs.github.com/en/actions/using-jobs/using-concurrency | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
cancel-in-progress: true | |
jobs: | |
code_quality_analyze: | |
name: code_quality_analyze | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-test" ] | |
steps: | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
CHECK_CONTAINER=infinity_build_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "CHECK_CONTAINER=${CHECK_CONTAINER}" >> $GITHUB_ENV | |
- name: Start builder container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${CHECK_CONTAINER} -e TZ=$TZ \ | |
-v $PWD:/infinity infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: Check format of changed C/C++ files | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} \ | |
| grep -E '\.(cpp|h|hpp|cppm)$' \ | |
| grep -v 'third_party/' \ | |
| grep -v 'network/' \ | |
| grep -v 'parser/' || true) | |
if [ -n "$CHANGED_FILES" ]; then | |
echo "Check format of changed C/C++ files with clang-format-20" | |
readarray -t files <<< "$CHANGED_FILES" | |
HAS_ERROR=0 | |
for file in "${files[@]}"; do | |
if [ -f "$file" ]; then | |
echo "Checking formatting: $file" | |
sudo docker exec ${CHECK_CONTAINER} bash -c "cd /infinity; clang-format-20 $file > ${file}.formatted; diff $file ${file}.formatted 2>/dev/null" | |
if [ $? -ne 0 ]; then | |
echo "❌ $file needs formatting" | |
HAS_ERROR=1 | |
else | |
echo "✅ $file is properly formatted" | |
fi | |
rm -f "${file}.formatted" | |
fi | |
done | |
if [ $HAS_ERROR -ne 0 ]; then | |
exit 1 | |
fi | |
else | |
echo "No C++ files changed" | |
fi | |
fi | |
- name: Stop container | |
if: always() | |
run: sudo docker stop ${CHECK_CONTAINER} | |
- name: Remove container | |
if: always() | |
run: sudo docker rm -f -v ${CHECK_CONTAINER} | |
debug_configure_and_build: | |
name: debug_configure_and_build | |
if: ${{ github.event_name != 'pull_request_target' || contains(github.event.pull_request_target.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-build" ] | |
needs: code_quality_analyze | |
outputs: | |
BUILD_IP: ${{ steps.get_ip.outputs.BUILD_IP }} | |
CONTAINER_NAME: ${{ steps.start.outputs.CONTAINER_NAME }} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
BUILDER_CONTAINER=infinity_build_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "CONTAINER_NAME=${BUILDER_CONTAINER}" >> $GITHUB_OUTPUT | |
echo "RUNNER_NAME=${RUNNER_NAME}" >> $GITHUB_OUTPUT | |
echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
- name: SSH to host server | |
id: ssh | |
uses: appleboy/[email protected] | |
with: | |
host: 172.17.0.1 | |
envs: BUILDER_CONTAINER, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
# stupid solution | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
capture_stdout: true | |
script: | | |
hostname -I | awk '{print $1}' | |
mkdir -p /home/infiniflow/artifacts/infinity/${BUILDER_CONTAINER} | |
- name: Get host ip | |
id: get_ip | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
read -r BUILD_IP _ <<< "${{ steps.ssh.outputs.stdout }}" | |
echo "Build ip is: ${BUILD_IP}" | |
echo "BUILD_IP=${BUILD_IP}" >> ${GITHUB_OUTPUT} | |
- name: Start builder container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ \ | |
-v $PWD:/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${BUILDER_CONTAINER}/:/infinity_artifact \ | |
infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: Configure debug version | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -fr cmake-build-debug && mkdir -p cmake-build-debug && cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -S /infinity -B /infinity/cmake-build-debug" | |
- name: Build debug version | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity && cmake --build /infinity/cmake-build-debug -j --target infinity" | |
- name: Build unit test | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" | |
cd /infinity && cmake --build /infinity/cmake-build-debug -j --target test_main" | |
- name: Prepare artifacts | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${BUILDER_CONTAINER} bash -c "mv /infinity/cmake-build-debug/src/infinity /infinity_artifact/ && | |
mv /infinity/cmake-build-debug/src/test_main /infinity_artifact/" | |
- name: Stop build container | |
if: always() | |
run: sudo docker stop ${BUILDER_CONTAINER} | |
- name: Remove build container | |
if: always() | |
run: sudo docker rm -f -v ${BUILDER_CONTAINER} | |
debug_sqllogical_test: | |
name: debug_sqllogical_test | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-test" ] | |
needs: debug_configure_and_build | |
outputs: | |
BUILD_CONTAINER_NAME: ${{ steps.start.outputs.BUILD_CONTAINER_NAME }} | |
TEST_CONTAINER_NAME: ${{ steps.start.outputs.TEST_CONTAINER_NAME }} | |
env: | |
BUILD_IP: ${{needs.debug_configure_and_build.outputs.BUILD_IP}} | |
CONTAINER_NAME: ${{needs.debug_configure_and_build.outputs.CONTAINER_NAME}} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TEST_CONTAINER_NAME=infinity_test_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "TEST_CONTAINER_NAME=${TEST_CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "BUILD_CONTAINER_NAME=${CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
- name: Download artifacts | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: CONTAINER_NAME, BUILD_IP | |
script: | | |
mkdir -p /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log | |
flock /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/lock -c "rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/infinity /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/" | |
- name: Prepare data dir & Start test container | |
# id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo mkdir -p /var/infinity/${TEST_CONTAINER_NAME}/ | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${TEST_CONTAINER_NAME} -e TZ=$TZ \ | |
-v $PWD:/infinity \ | |
-v /var/infinity/${TEST_CONTAINER_NAME}/:/var/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/infinity:/infinity/cmake-build-debug/src/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log:/infinity_log/ \ | |
infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: SSH to host server | |
id: ssh | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: TEST_CONTAINER_NAME, BUILD_CONTAINER_NAME, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
script: | | |
sudo docker exec ${TEST_CONTAINER_NAME} mkdir -p /infinity/cmake-build-debug/src | |
- name: Install pysdk | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "rm -rf /root/.config/pip/pip.conf && | |
cd /infinity/ && | |
pip3 uninstall -y infinity-sdk && | |
pip3 install python/infinity_sdk -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
- name: Start infinity debug version | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
cmake-build-debug/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > sqllogictest_debug.log 2> sqllogictest_debug_error.log" & | |
- name: Run sqllogictest debug version | |
if: ${{ !cancelled() && !failure() }} | |
id: run_tests_debug | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
LD_PRELOAD=/usr/local/lib/clang/20/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/sqllogictest.py" | |
sleep 1s | |
- name: Collect infinity log | |
if: ${{ !cancelled() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c \ | |
"mv /infinity/sqllogictest_debug.log /infinity_log/sqllogictest_debug.log && | |
mv /infinity/sqllogictest_debug_error.log /infinity_log/sqllogictest_debug_error.log && | |
echo 'sqllogictest debug log' && | |
tail -n 1000 /infinity_log/sqllogictest_debug.log && | |
echo 'sqllogictest debug error log' && | |
tail -n 1000 /infinity_log/sqllogictest_debug_error.log" | |
- name: Remove test container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${TEST_CONTAINER_NAME} | |
sudo rm -rf /var/infinity/${TEST_CONTAINER_NAME}/ | |
debug_unit_test: | |
name: debug_unit_test | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-test" ] | |
needs: debug_configure_and_build | |
outputs: | |
BUILD_CONTAINER_NAME: ${{ steps.start.outputs.BUILD_CONTAINER_NAME }} | |
TEST_CONTAINER_NAME: ${{ steps.start.outputs.TEST_CONTAINER_NAME }} | |
env: | |
BUILD_IP: ${{needs.debug_configure_and_build.outputs.BUILD_IP}} | |
CONTAINER_NAME: ${{needs.debug_configure_and_build.outputs.CONTAINER_NAME}} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TEST_CONTAINER_NAME=infinity_test_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "TEST_CONTAINER_NAME=${TEST_CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "BUILD_CONTAINER_NAME=${CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
- name: Download artifacts | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: CONTAINER_NAME, BUILD_IP | |
script: | | |
mkdir -p /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log | |
flock /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/lock -c \ | |
"rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/test_main /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/" | |
- name: Prepare data dir & Start test container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo mkdir -p /var/infinity/${TEST_CONTAINER_NAME}/ | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${TEST_CONTAINER_NAME} -e TZ=$TZ \ | |
-v $PWD:/infinity \ | |
-v /var/infinity/${TEST_CONTAINER_NAME}/:/var/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/test_main:/infinity/cmake-build-debug/src/test_main \ | |
-v /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log:/infinity_log/ \ | |
infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: Start minio container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
MINIO_DIR=~/minio_data_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> $GITHUB_ENV | |
echo "MINIO_DIR=${MINIO_DIR}" >> $GITHUB_ENV | |
sudo mkdir ${MINIO_DIR} | |
sudo docker run -d --net=container:${TEST_CONTAINER_NAME} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v ${MINIO_DIR}:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005" | |
sleep 5s | |
if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then | |
echo "Minio container is not running" | |
exit 1 | |
fi | |
- name: SSH to host server to start test task | |
id: ssh | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: TEST_CONTAINER_NAME, BUILD_CONTAINER_NAME, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
script: | | |
sudo docker exec ${TEST_CONTAINER_NAME} mkdir -p /infinity/cmake-build-debug/src | |
- name: Unit test debug version | |
if: ${{ !cancelled() && !failure() }} | |
id: run_unit_test_debug | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
ASAN_OPTIONS=detect_leaks=0 cmake-build-debug/src/test_main --gtest_also_run_disabled_tests --gtest_filter=-*SLOW_* > unittest_debug.log 2> unittest_debug_error.log" | |
- name: Collect infinity log | |
if: ${{ !cancelled() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c \ | |
"mv /infinity/unittest_debug.log /infinity_log/unittest_debug.log && | |
mv /infinity/unittest_debug_error.log /infinity_log/unittest_debug_error.log && | |
echo 'ut debug log' && | |
tail -n 1000 /infinity_log/unittest_debug.log && | |
echo 'ut debug error log' && | |
tail -n 1000 /infinity_log/unittest_debug_error.log" | |
- name: Remove test container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${TEST_CONTAINER_NAME} | |
sudo rm -rf /var/infinity/${TEST_CONTAINER_NAME}/ | |
- name: Remove minio container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${MINIO_CONTAINER} | |
debug_restart_test: | |
name: debug_restart_test | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-test" ] | |
needs: debug_configure_and_build | |
outputs: | |
BUILD_CONTAINER_NAME: ${{ steps.start.outputs.BUILD_CONTAINER_NAME }} | |
TEST_CONTAINER_NAME: ${{ steps.start.outputs.TEST_CONTAINER_NAME }} | |
env: | |
BUILD_IP: ${{needs.debug_configure_and_build.outputs.BUILD_IP}} | |
CONTAINER_NAME: ${{needs.debug_configure_and_build.outputs.CONTAINER_NAME}} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TEST_CONTAINER_NAME=infinity_test_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "TEST_CONTAINER_NAME=${TEST_CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "BUILD_CONTAINER_NAME=${CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
- name: Prepare data dir | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: CONTAINER_NAME | |
script: | | |
sudo mkdir -p /var/infinity/${CONTAINER_NAME}/ | |
- name: Download artifacts | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: CONTAINER_NAME, BUILD_IP | |
script: | | |
mkdir -p /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log | |
flock /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/lock -c \ | |
"rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/infinity /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/" | |
- name: Start test container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo mkdir -p /var/infinity/${TEST_CONTAINER_NAME}/ | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${TEST_CONTAINER_NAME} -e TZ=$TZ \ | |
-v $PWD:/infinity \ | |
-v /var/infinity/${TEST_CONTAINER_NAME}/:/var/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/infinity:/infinity/cmake-build-debug/src/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log:/infinity_log/ \ | |
infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: SSH to host server | |
id: ssh | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: TEST_CONTAINER_NAME, BUILD_CONTAINER_NAME, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
script: | | |
sudo docker exec ${TEST_CONTAINER_NAME} mkdir -p /infinity/cmake-build-debug/src | |
- name: Install pysdk | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "rm -rf /root/.config/pip/pip.conf && | |
cd /infinity/ && | |
pip3 uninstall -y infinity-sdk && | |
pip3 install python/infinity_sdk -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
- name: Prepare restart test data | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME} | |
echo "RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX}" >> $GITHUB_ENV | |
- name: Run restart test | |
if: ${{ !cancelled() && !failure() }} | |
id: run_restart_test | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
python3 tools/run_restart_test.py --infinity_path=cmake-build-debug/src/infinity" | |
- name: Collect infinity log | |
if: ${{ !cancelled() }} | |
run: | | |
sudo python3 scripts/collect_restart_log.py --executable_path=cmake-build-debug/src/infinity --output_dir=/infinity_log --failure=${failure} | |
- name: Remove test container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${TEST_CONTAINER_NAME} | |
sudo rm -rf /var/infinity/${TEST_CONTAINER_NAME}/ | |
debug_pysdk_remote_test: | |
name: debug_pysdk_remote_test | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-test" ] | |
needs: debug_configure_and_build | |
outputs: | |
BUILD_CONTAINER_NAME: ${{ steps.start.outputs.BUILD_CONTAINER_NAME }} | |
TEST_CONTAINER_NAME: ${{ steps.start.outputs.TEST_CONTAINER_NAME }} | |
env: | |
BUILD_IP: ${{needs.debug_configure_and_build.outputs.BUILD_IP}} | |
CONTAINER_NAME: ${{needs.debug_configure_and_build.outputs.CONTAINER_NAME}} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TEST_CONTAINER_NAME=infinity_test_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "TEST_CONTAINER_NAME=${TEST_CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "BUILD_CONTAINER_NAME=${CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
- name: Download artifacts | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: CONTAINER_NAME, BUILD_IP | |
script: | | |
mkdir -p /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log | |
flock /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/lock -c "rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/infinity /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/" | |
- name: Prepare data dir & Start test container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo mkdir -p /var/infinity/${TEST_CONTAINER_NAME}/ | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${TEST_CONTAINER_NAME} -e TZ=$TZ \ | |
-v $PWD:/infinity \ | |
-v /var/infinity/${TEST_CONTAINER_NAME}/:/var/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/infinity:/infinity/cmake-build-debug/src/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log:/infinity_log/ \ | |
infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: SSH to host server | |
id: ssh | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: TEST_CONTAINER_NAME, BUILD_CONTAINER_NAME, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
script: | | |
sudo docker exec ${TEST_CONTAINER_NAME} mkdir -p /infinity/cmake-build-debug/src | |
- name: Install pysdk | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "rm -rf /root/.config/pip/pip.conf && | |
cd /infinity/ && | |
pip3 uninstall -y infinity-sdk && | |
pip3 install python/infinity_sdk -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
- name: Start infinity debug version | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
cmake-build-debug/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > pysdk_remote_debug.log 2> pysdk_remote_debug_error.log" & | |
- name: Run pysdk remote infinity test debug version | |
if: ${{ !cancelled() && !failure() }} | |
id: run_pysdk_remote_debug | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
LD_PRELOAD=/usr/local/lib/clang/20/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_pysdk_remote_infinity_test.py" | |
sleep 1s | |
- name: Collect infinity log | |
if: ${{ !cancelled() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c \ | |
"mv /infinity/pysdk_remote_debug.log /infinity_log/pysdk_remote_debug.log && | |
mv /infinity/pysdk_remote_debug_error.log /infinity_log/pysdk_remote_debug_error.log && | |
echo 'pysdk remote debug log' && | |
tail -n 1000 /infinity_log/pysdk_remote_debug.log && | |
echo 'pysdk remote debug error log' && | |
tail -n 1000 /infinity_log/pysdk_remote_debug_error.log" | |
- name: Remove test container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${TEST_CONTAINER_NAME} | |
sudo rm -rf /var/infinity/${TEST_CONTAINER_NAME}/ | |
debug_http_test: | |
name: debug_http_test | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-test" ] | |
needs: debug_configure_and_build | |
outputs: | |
BUILD_CONTAINER_NAME: ${{ steps.start.outputs.BUILD_CONTAINER_NAME }} | |
TEST_CONTAINER_NAME: ${{ steps.start.outputs.TEST_CONTAINER_NAME }} | |
env: | |
BUILD_IP: ${{needs.debug_configure_and_build.outputs.BUILD_IP}} | |
CONTAINER_NAME: ${{needs.debug_configure_and_build.outputs.CONTAINER_NAME}} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TEST_CONTAINER_NAME=infinity_test_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "TEST_CONTAINER_NAME=${TEST_CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "BUILD_CONTAINER_NAME=${CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
- name: Download artifacts | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: CONTAINER_NAME, BUILD_IP | |
script: | | |
mkdir -p /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log | |
flock /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/lock -c \ | |
"rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/infinity /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/" | |
- name: Prepare data dir & Start test container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo mkdir -p /var/infinity/${TEST_CONTAINER_NAME}/ | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${TEST_CONTAINER_NAME} -e TZ=$TZ \ | |
-v $PWD:/infinity \ | |
-v /var/infinity/${TEST_CONTAINER_NAME}/:/var/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/infinity:/infinity/cmake-build-debug/src/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log:/infinity_log/ \ | |
infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: SSH to host server | |
id: ssh | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: TEST_CONTAINER_NAME, BUILD_CONTAINER_NAME, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
script: | | |
sudo docker exec ${TEST_CONTAINER_NAME} mkdir -p /infinity/cmake-build-debug/src | |
sudo cp /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/infinity ${RUNNER_WORKSPACE_PREFIX}/${RUNNER_NAME}/infinity/infinity/cmake-build-debug/src/ | |
- name: Install pysdk | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "rm -rf /root/.config/pip/pip.conf | |
cd /infinity/ | |
pip3 uninstall -y infinity-sdk | |
pip3 install python/infinity_sdk -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
- name: Start infinity debug version | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ | |
cmake-build-debug/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > http_remote_debug.log 2> http_remote_debug_error.log" & | |
- name: Run HTTP infinity test debug version | |
if: ${{ !cancelled() && !failure() }} | |
id: run_http_remote_debug | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ | |
LD_PRELOAD=/usr/local/lib/clang/20/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_http_api.py" | |
sleep 1s | |
- name: Collect infinity log | |
if: ${{ !cancelled() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c \ | |
"mv /infinity/http_remote_debug.log /infinity_log/http_remote_debug.log && | |
mv /infinity/http_remote_debug_error.log /infinity_log/http_remote_debug_error.log && | |
echo 'http remote debug log' && | |
tail -n 1000 /infinity_log/http_remote_debug.log && | |
echo 'http remote debug error log' && | |
tail -n 1000 /infinity_log/http_remote_debug_error.log" | |
- name: Remove test container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${TEST_CONTAINER_NAME} | |
sudo rm -rf /var/infinity/${TEST_CONTAINER_NAME}/ | |
debug_pysdk_parallel_test: | |
name: debug_pysdk_parallel_test | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-test" ] | |
needs: | |
- debug_configure_and_build | |
outputs: | |
BUILD_CONTAINER_NAME: ${{ steps.start.outputs.BUILD_CONTAINER_NAME }} | |
TEST_CONTAINER_NAME: ${{ steps.start.outputs.TEST_CONTAINER_NAME }} | |
env: | |
BUILD_IP: ${{needs.debug_configure_and_build.outputs.BUILD_IP}} | |
CONTAINER_NAME: ${{needs.debug_configure_and_build.outputs.CONTAINER_NAME}} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TEST_CONTAINER_NAME=infinity_test_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "TEST_CONTAINER_NAME=${TEST_CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "BUILD_CONTAINER_NAME=${CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
- name: Download artifacts | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: CONTAINER_NAME, BUILD_IP | |
script: | | |
mkdir -p /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log | |
flock /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/lock -c \ | |
"rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/infinity /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/" | |
- name: Prepare data dir & Start test container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo mkdir -p /var/infinity/${TEST_CONTAINER_NAME}/ | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${TEST_CONTAINER_NAME} -e TZ=$TZ \ | |
-v $PWD:/infinity \ | |
-v /var/infinity/${TEST_CONTAINER_NAME}/:/var/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/infinity:/infinity/cmake-build-debug/src/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log:/infinity_log/ \ | |
infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: SSH to host server | |
id: ssh | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: TEST_CONTAINER_NAME, BUILD_CONTAINER_NAME, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
script: | | |
sudo docker exec ${TEST_CONTAINER_NAME} mkdir -p /infinity/cmake-build-debug/src | |
- name: Install pysdk | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "rm -rf /root/.config/pip/pip.conf | |
cd /infinity/ | |
pip3 uninstall -y infinity-sdk | |
pip3 install python/infinity_sdk -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
- name: Start infinity debug version | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
cmake-build-debug/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > parallel_test_debug.log 2> parallel_test_debug_error.log" & | |
- name: Run pysdk parallel test debug version | |
if: ${{ !cancelled() && !failure() }} | |
id: run_parallel_test_debug | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ | |
LD_PRELOAD=/usr/local/lib/clang/20/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_parallel_test.py" | |
sleep 1s | |
- name: Collect infinity log | |
if: ${{ !cancelled() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c \ | |
"mv /infinity/parallel_test_debug.log /infinity_log/parallel_test_debug.log && | |
mv /infinity/parallel_test_debug_error.log /infinity_log/parallel_test_debug_error.log && | |
echo 'parallel test debug log' && | |
tail -n 1000 /infinity_log/parallel_test_debug.log && | |
echo 'parallel test debug error log' && | |
tail -n 1000 /infinity_log/parallel_test_debug_error.log" | |
- name: Remove test container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${TEST_CONTAINER_NAME} | |
sudo rm -rf /var/infinity/${TEST_CONTAINER_NAME}/ | |
release_configure_and_build: | |
name: release_configure_and_build | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-build" ] | |
needs: code_quality_analyze | |
outputs: | |
BUILD_IP: ${{ steps.get_ip.outputs.BUILD_IP }} | |
CONTAINER_NAME: ${{ steps.start.outputs.CONTAINER_NAME }} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
BUILDER_CONTAINER=infinity_build_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "CONTAINER_NAME=${BUILDER_CONTAINER}" >> $GITHUB_OUTPUT | |
echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
- name: SSH to host server | |
id: ssh | |
uses: appleboy/[email protected] | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
capture_stdout: true | |
envs: BUILDER_CONTAINER, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
script: | | |
hostname -I | awk '{print $1}' | |
mkdir -p /home/infiniflow/artifacts/infinity/${BUILDER_CONTAINER} | |
- name: Get host ip | |
id: get_ip | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
read -r BUILD_IP _ <<< "${{ steps.ssh.outputs.stdout }}" | |
echo "Build ip is: ${BUILD_IP}" | |
echo "BUILD_IP=${BUILD_IP}" >> ${GITHUB_OUTPUT} | |
- name: Start builder container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} \ | |
-v $PWD:/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${BUILDER_CONTAINER}/:/infinity_artifact \ | |
infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: Configure release version | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" | |
cd /infinity && rm -fr cmake-build-release | |
mkdir -p cmake-build-release | |
cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -S /infinity -B /infinity/cmake-build-release" | |
- name: Build release version | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity | |
cmake --build /infinity/cmake-build-release -j --target infinity" | |
- name: Build unit test | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" | |
cd /infinity && cmake --build /infinity/cmake-build-release -j --target test_main knn_import_benchmark knn_query_benchmark" | |
- name: Prepare artifacts | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${BUILDER_CONTAINER} bash -c "mv /infinity/cmake-build-release/src/infinity /infinity_artifact/ && | |
mv /infinity/cmake-build-release/src/test_main /infinity_artifact/ && | |
mv /infinity/cmake-build-release/benchmark/local_infinity/knn_query_benchmark /infinity_artifact/ && | |
mv /infinity/cmake-build-release/benchmark/local_infinity/knn_import_benchmark /infinity_artifact/" | |
- name: Stop build container | |
if: always() | |
run: sudo docker stop ${BUILDER_CONTAINER} | |
- name: Remove build container | |
if: always() | |
run: sudo docker rm -f -v ${BUILDER_CONTAINER} | |
release_sqllogical_test: | |
name: release_sqllogical_test | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-test" ] | |
needs: release_configure_and_build | |
outputs: | |
BUILD_CONTAINER_NAME: ${{ steps.start.outputs.BUILD_CONTAINER_NAME }} | |
TEST_CONTAINER_NAME: ${{ steps.start.outputs.TEST_CONTAINER_NAME }} | |
env: | |
BUILD_IP: ${{needs.release_configure_and_build.outputs.BUILD_IP}} | |
CONTAINER_NAME: ${{needs.release_configure_and_build.outputs.CONTAINER_NAME}} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TEST_CONTAINER_NAME=infinity_test_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "TEST_CONTAINER_NAME=${TEST_CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "BUILD_CONTAINER_NAME=${CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
- name: Download artifacts | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: CONTAINER_NAME, BUILD_IP | |
script: | | |
mkdir -p /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log | |
flock /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/lock -c \ | |
"rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/infinity /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/" | |
- name: Prepare data dir & Start test container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo mkdir -p /var/infinity/${TEST_CONTAINER_NAME}/ | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${TEST_CONTAINER_NAME} -e TZ=$TZ \ | |
-v $PWD:/infinity \ | |
-v /var/infinity/${TEST_CONTAINER_NAME}/:/var/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/infinity:/infinity/cmake-build-release/src/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log:/infinity_log/ \ | |
infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: SSH to host server | |
id: ssh | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: TEST_CONTAINER_NAME, BUILD_CONTAINER_NAME, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
script: | | |
sudo docker exec ${TEST_CONTAINER_NAME} mkdir -p /infinity/cmake-build-release/src | |
- name: Install pysdk | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "rm -rf /root/.config/pip/pip.conf && | |
cd /infinity/ && | |
pip3 uninstall -y infinity-sdk && | |
pip3 install python/infinity_sdk -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
- name: Start infinity release version | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
cmake-build-release/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > sqllogictest_release.log 2> sqllogictest_release_error.log" & | |
- name: Run sqllogictest release version | |
if: ${{ !cancelled() && !failure() }} | |
id: run_tests_debug | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
LD_PRELOAD=/usr/local/lib/clang/20/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/sqllogictest.py" | |
sleep 1s | |
- name: Collect infinity log | |
if: ${{ !cancelled() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c \ | |
"mv /infinity/sqllogictest_release.log /infinity_log/sqllogictest_release.log && | |
mv /infinity/sqllogictest_release_error.log /infinity_log/sqllogictest_release_error.log && | |
echo 'sqllogictest release log' && | |
tail -n 1000 /infinity_log/sqllogictest_release.log && | |
echo 'sqllogictest release error log' && | |
tail -n 1000 /infinity_log/sqllogictest_release_error.log" | |
- name: Remove test container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${TEST_CONTAINER_NAME} | |
sudo rm -rf /var/infinity/${TEST_CONTAINER_NAME}/ | |
release_unit_test: | |
name: release_unit_test | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-test" ] | |
needs: release_configure_and_build | |
outputs: | |
BUILD_CONTAINER_NAME: ${{ steps.start.outputs.BUILD_CONTAINER_NAME }} | |
TEST_CONTAINER_NAME: ${{ steps.start.outputs.TEST_CONTAINER_NAME }} | |
env: | |
BUILD_IP: ${{needs.release_configure_and_build.outputs.BUILD_IP}} | |
CONTAINER_NAME: ${{needs.release_configure_and_build.outputs.CONTAINER_NAME}} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TEST_CONTAINER_NAME=infinity_test_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "TEST_CONTAINER_NAME=${TEST_CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "BUILD_CONTAINER_NAME=${CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
- name: Download artifacts | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: CONTAINER_NAME, BUILD_IP | |
script: | | |
mkdir -p /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log | |
flock /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/ut_lock -c \ | |
"rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/test_main /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/" | |
- name: Prepare data dir & Start test container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo mkdir -p /var/infinity/${TEST_CONTAINER_NAME}/ | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${TEST_CONTAINER_NAME} -e TZ=$TZ \ | |
-v $PWD:/infinity \ | |
-v /var/infinity/${TEST_CONTAINER_NAME}/:/var/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/test_main:/infinity/cmake-build-release/src/test_main \ | |
-v /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log:/infinity_log/ \ | |
infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: Start minio container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
MINIO_DIR=~/minio_data_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> $GITHUB_ENV | |
echo "MINIO_DIR=${MINIO_DIR}" >> $GITHUB_ENV | |
sudo mkdir ${MINIO_DIR} | |
sudo docker run -d --net=container:${TEST_CONTAINER_NAME} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v ${MINIO_DIR}:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005" | |
sleep 5s | |
if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then | |
echo "Minio container is not running" | |
exit 1 | |
fi | |
- name: SSH to host server to start test task | |
id: ssh | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: TEST_CONTAINER_NAME, BUILD_CONTAINER_NAME, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
script: | | |
sudo docker exec ${TEST_CONTAINER_NAME} mkdir -p /infinity/cmake-build-release/src | |
- name: Unit test release version | |
if: ${{ !cancelled() && !failure() }} | |
id: run_unit_test_debug | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
ASAN_OPTIONS=detect_leaks=0 cmake-build-release/src/test_main --gtest_also_run_disabled_tests --gtest_filter=-*SLOW_* > unittest_release.log 2> unittest_release_error.log" | |
- name: Collect infinity log | |
if: ${{ !cancelled() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c \ | |
"mv /infinity/unittest_release.log /infinity_log/unittest_release.log && | |
mv /infinity/unittest_release_error.log /infinity_log/unittest_release_error.log && | |
echo 'ut release log' && | |
tail -n 1000 /infinity_log/unittest_release.log && | |
echo 'ut release error log' && | |
tail -n 1000 /infinity_log/unittest_release_error.log" | |
- name: Remove test container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${TEST_CONTAINER_NAME} | |
sudo rm -rf /var/infinity/${TEST_CONTAINER_NAME}/ | |
- name: Remove minio container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${MINIO_CONTAINER} | |
release_pysdk_remote_test: | |
name: release_pysdk_remote_test | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-test" ] | |
needs: release_configure_and_build | |
outputs: | |
BUILD_CONTAINER_NAME: ${{ steps.start.outputs.BUILD_CONTAINER_NAME }} | |
TEST_CONTAINER_NAME: ${{ steps.start.outputs.TEST_CONTAINER_NAME }} | |
env: | |
BUILD_IP: ${{needs.release_configure_and_build.outputs.BUILD_IP}} | |
CONTAINER_NAME: ${{needs.release_configure_and_build.outputs.CONTAINER_NAME}} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TEST_CONTAINER_NAME=infinity_test_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "TEST_CONTAINER_NAME=${TEST_CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "BUILD_CONTAINER_NAME=${CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
- name: Download artifacts | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: CONTAINER_NAME, BUILD_IP | |
script: | | |
mkdir -p /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log | |
flock /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/lock -c \ | |
"rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/infinity /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/" | |
- name: Prepare data dir & Start test container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo mkdir -p /var/infinity/${TEST_CONTAINER_NAME}/ | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${TEST_CONTAINER_NAME} -e TZ=$TZ \ | |
-v $PWD:/infinity \ | |
-v /var/infinity/${TEST_CONTAINER_NAME}/:/var/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/infinity:/infinity/cmake-build-release/src/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log:/infinity_log/ \ | |
infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: SSH to host server | |
id: ssh | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: TEST_CONTAINER_NAME, BUILD_CONTAINER_NAME, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
script: | | |
sudo docker exec ${TEST_CONTAINER_NAME} mkdir -p /infinity/cmake-build-release/src | |
- name: Install pysdk | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "rm -rf /root/.config/pip/pip.conf && | |
cd /infinity/ && | |
pip3 uninstall -y infinity-sdk && | |
pip3 install python/infinity_sdk -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
- name: Start infinity release version | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
cmake-build-release/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > pysdk_remote_release.log 2> pysdk_remote_release_error.log" & | |
- name: Run pysdk remote infinity test release version | |
if: ${{ !cancelled() && !failure() }} | |
id: run_pysdk_remote_debug | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
LD_PRELOAD=/usr/local/lib/clang/20/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_pysdk_remote_infinity_test.py" | |
sleep 1s | |
- name: Collect infinity log | |
if: ${{ !cancelled() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c \ | |
"mv /infinity/pysdk_remote_release.log /infinity_log/pysdk_remote_release.log && | |
mv /infinity/pysdk_remote_release_error.log /infinity_log/pysdk_remote_release_error.log && | |
echo 'pysdk remote release log' && | |
tail -n 1000 /infinity_log/pysdk_remote_release.log && | |
echo 'pysdk remote release error log' && | |
tail -n 1000 /infinity_log/pysdk_remote_release_error.log" | |
- name: Remove builder container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${TEST_CONTAINER_NAME} | |
sudo rm -rf /var/infinity/${TEST_CONTAINER_NAME}/ | |
release_http_test: | |
name: release_http_test | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-test" ] | |
needs: release_configure_and_build | |
outputs: | |
BUILD_CONTAINER_NAME: ${{ steps.start.outputs.BUILD_CONTAINER_NAME }} | |
TEST_CONTAINER_NAME: ${{ steps.start.outputs.TEST_CONTAINER_NAME }} | |
env: | |
BUILD_IP: ${{needs.release_configure_and_build.outputs.BUILD_IP}} | |
CONTAINER_NAME: ${{needs.release_configure_and_build.outputs.CONTAINER_NAME}} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TEST_CONTAINER_NAME=infinity_test_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "TEST_CONTAINER_NAME=${TEST_CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "BUILD_CONTAINER_NAME=${CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
- name: Download artifacts | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: CONTAINER_NAME, BUILD_IP | |
script: | | |
mkdir -p /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log | |
flock /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/lock -c \ | |
"rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/infinity /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/" | |
- name: Prepare data dir & Start test container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo mkdir -p /var/infinity/${TEST_CONTAINER_NAME}/ | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${TEST_CONTAINER_NAME} -e TZ=$TZ \ | |
-v $PWD:/infinity \ | |
-v /var/infinity/${TEST_CONTAINER_NAME}/:/var/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/infinity:/infinity/cmake-build-release/src/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log:/infinity_log/ \ | |
infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: SSH to host server to start test task | |
id: ssh | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: TEST_CONTAINER_NAME, BUILD_CONTAINER_NAME, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
script: | | |
sudo docker exec ${TEST_CONTAINER_NAME} mkdir -p /infinity/cmake-build-release/src | |
- name: Install pysdk | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "rm -rf /root/.config/pip/pip.conf && | |
cd /infinity/ && | |
pip3 uninstall -y infinity-sdk && | |
pip3 install python/infinity_sdk -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
- name: Start infinity release version | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
cmake-build-release/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > http_remote_release.log 2> http_remote_release_error.log" & | |
- name: Run HTTP infinity test release version | |
if: ${{ !cancelled() && !failure() }} | |
id: run_http_remote_debug | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
LD_PRELOAD=/usr/local/lib/clang/20/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_http_api.py" | |
sleep 1s | |
- name: Collect infinity log | |
if: ${{ !cancelled() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c \ | |
"mv /infinity/http_remote_release.log /infinity_log/http_remote_release.log && | |
mv /infinity/http_remote_release_error.log /infinity_log/http_remote_release_error.log && | |
echo 'http remote release log' && | |
tail -n 1000 /infinity_log/http_remote_release.log && | |
echo 'http remote release error log' && | |
tail -n 1000 /infinity_log/http_remote_release_error.log" | |
- name: Remove test container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${TEST_CONTAINER_NAME} | |
sudo rm -rf /var/infinity/${TEST_CONTAINER_NAME}/ | |
release_pysdk_parallel_test: | |
name: release_pysdk_parallel_test | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-test" ] | |
needs: release_configure_and_build | |
outputs: | |
BUILD_CONTAINER_NAME: ${{ steps.start.outputs.BUILD_CONTAINER_NAME }} | |
TEST_CONTAINER_NAME: ${{ steps.start.outputs.TEST_CONTAINER_NAME }} | |
env: | |
BUILD_IP: ${{needs.release_configure_and_build.outputs.BUILD_IP}} | |
CONTAINER_NAME: ${{needs.release_configure_and_build.outputs.CONTAINER_NAME}} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TEST_CONTAINER_NAME=infinity_test_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "TEST_CONTAINER_NAME=${TEST_CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "BUILD_CONTAINER_NAME=${CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
- name: Download artifacts | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: CONTAINER_NAME, BUILD_IP | |
script: | | |
mkdir -p /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log | |
flock /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/lock -c \ | |
"rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/infinity /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/" | |
- name: Prepare data dir & Start test container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo mkdir -p /var/infinity/${TEST_CONTAINER_NAME}/ | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${TEST_CONTAINER_NAME} -e TZ=$TZ \ | |
-v $PWD:/infinity \ | |
-v /var/infinity/${TEST_CONTAINER_NAME}/:/var/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/infinity:/infinity/cmake-build-release/src/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log:/infinity_log/ \ | |
infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: SSH to host server | |
id: ssh | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: TEST_CONTAINER_NAME, BUILD_CONTAINER_NAME, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
script: | | |
sudo docker exec ${TEST_CONTAINER_NAME} mkdir -p /infinity/cmake-build-release/src | |
- name: Install pysdk | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "rm -rf /root/.config/pip/pip.conf && | |
cd /infinity/ && | |
pip3 uninstall -y infinity-sdk && | |
pip3 install python/infinity_sdk -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
- name: Start infinity release version | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
cmake-build-release/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > parallel_test_release.log 2> parallel_test_release_error.log" & | |
- name: Run pysdk parallel test release version | |
if: ${{ !cancelled() && !failure() }} | |
id: run_parallel_test_debug | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
LD_PRELOAD=/usr/local/lib/clang/20/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_parallel_test.py" | |
sleep 1s | |
- name: Collect infinity log | |
if: ${{ !cancelled() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c \ | |
"mv /infinity/parallel_test_release.log /infinity_log/parallel_test_release.log && | |
mv /infinity/parallel_test_release_error.log /infinity_log/parallel_test_release_error.log && | |
echo 'pysdk parallel release log' && | |
tail -n 1000 /infinity_log/parallel_test_release.log && | |
echo 'pysdk parallel release error log' && | |
tail -n 1000 /infinity_log/parallel_test_release_error.log" | |
- name: Remove test container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${TEST_CONTAINER_NAME} | |
sudo rm -rf /var/infinity/${TEST_CONTAINER_NAME}/ | |
release_minio_pysdk_remote_test: | |
name: release_minio_pysdk_remote_test | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-test" ] | |
needs: release_configure_and_build | |
outputs: | |
BUILD_CONTAINER_NAME: ${{ steps.start.outputs.BUILD_CONTAINER_NAME }} | |
TEST_CONTAINER_NAME: ${{ steps.start.outputs.TEST_CONTAINER_NAME }} | |
env: | |
BUILD_IP: ${{needs.release_configure_and_build.outputs.BUILD_IP}} | |
CONTAINER_NAME: ${{needs.release_configure_and_build.outputs.CONTAINER_NAME}} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TEST_CONTAINER_NAME=infinity_test_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "TEST_CONTAINER_NAME=${TEST_CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "BUILD_CONTAINER_NAME=${CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
- name: Download artifacts | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: CONTAINER_NAME, BUILD_IP | |
script: | | |
mkdir -p /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log | |
flock /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/lock -c \ | |
"rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/infinity /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/" | |
- name: Prepare data dir & Start test container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo mkdir -p /var/infinity/${TEST_CONTAINER_NAME}/ | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${TEST_CONTAINER_NAME} -e TZ=$TZ \ | |
-v $PWD:/infinity \ | |
-v /var/infinity/${TEST_CONTAINER_NAME}/:/var/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/infinity:/infinity/cmake-build-release/src/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log:/infinity_log/ \ | |
infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: SSH to host server to start test task | |
id: ssh | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: TEST_CONTAINER_NAME, BUILD_CONTAINER_NAME, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
script: | | |
sudo docker exec ${TEST_CONTAINER_NAME} mkdir -p /infinity/cmake-build-release/src | |
- name: Install pysdk | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "rm -rf /root/.config/pip/pip.conf && | |
cd /infinity/ && | |
pip3 uninstall -y infinity-sdk && | |
pip3 install python/infinity_sdk -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
- name: Start minio container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
MINIO_DIR=~/minio_data_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> $GITHUB_ENV | |
echo "MINIO_DIR=${MINIO_DIR}" >> $GITHUB_ENV | |
sudo mkdir ${MINIO_DIR} | |
sudo docker run -d --net=container:${TEST_CONTAINER_NAME} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v ${MINIO_DIR}:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005" | |
sleep 5s | |
if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then | |
echo "Minio container is not running" | |
exit 1 | |
fi | |
- name: Start infinity release version | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
cmake-build-release/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > minio_pysdk_remote_test_release.log 2> minio_pysdk_remote_test_release_error.log" & | |
- name: Run minio pysdk remote test | |
if: ${{ !cancelled() && !failure() }} | |
id: run_minio_pysdk_remote_release | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
python3 tools/run_pysdk_remote_infinity_test.py" | |
sleep 1s | |
- name: Collect infinity log | |
if: ${{ !cancelled() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c \ | |
"mv /infinity/minio_pysdk_remote_test_release.log /infinity_log/minio_pysdk_remote_test_release.log && | |
mv /infinity/minio_pysdk_remote_test_release_error.log /infinity_log/minio_pysdk_remote_test_release_error.log && | |
echo 'minio pysdk remote test release log' && | |
tail -n 1000 /infinity_log/minio_pysdk_remote_test_release.log && | |
echo 'minio pysdk remote test release error log' && | |
tail -n 1000 /infinity_log/minio_pysdk_remote_test_release_error.log" | |
- name: Remove test container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${TEST_CONTAINER_NAME} | |
sudo rm -rf /var/infinity/${TEST_CONTAINER_NAME}/ | |
- name: Remove minio container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${MINIO_CONTAINER} | |
release_minio_parallel_test: | |
name: release_minio_parallel_test | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "infinity-test" ] | |
needs: release_configure_and_build | |
outputs: | |
BUILD_CONTAINER_NAME: ${{ steps.start.outputs.BUILD_CONTAINER_NAME }} | |
TEST_CONTAINER_NAME: ${{ steps.start.outputs.TEST_CONTAINER_NAME }} | |
env: | |
BUILD_IP: ${{needs.release_configure_and_build.outputs.BUILD_IP}} | |
CONTAINER_NAME: ${{needs.release_configure_and_build.outputs.CONTAINER_NAME}} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Prepare env variable | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TEST_CONTAINER_NAME=infinity_test_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "TEST_CONTAINER_NAME=${TEST_CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "BUILD_CONTAINER_NAME=${CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
- name: Download artifacts | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: CONTAINER_NAME, BUILD_IP | |
script: | | |
mkdir -p /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log | |
flock /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/lock -c \ | |
"rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/infinity /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/" | |
- name: Prepare data dir & Start test container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo mkdir -p /var/infinity/${TEST_CONTAINER_NAME}/ | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${TEST_CONTAINER_NAME} -e TZ=$TZ \ | |
-v $PWD:/infinity \ | |
-v /var/infinity/${TEST_CONTAINER_NAME}/:/var/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/infinity:/infinity/cmake-build-release/src/infinity \ | |
-v /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/log:/infinity_log/ \ | |
infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: SSH to host server | |
id: ssh | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: TEST_CONTAINER_NAME, BUILD_CONTAINER_NAME, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
script: | | |
sudo docker exec ${TEST_CONTAINER_NAME} mkdir -p /infinity/cmake-build-release/src | |
- name: Install pysdk | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "rm -rf /root/.config/pip/pip.conf && | |
cd /infinity/ && | |
pip3 uninstall -y infinity-sdk && | |
pip3 install python/infinity_sdk -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
- name: Start minio container | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
MINIO_DIR=~/minio_data_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> $GITHUB_ENV | |
echo "MINIO_DIR=${MINIO_DIR}" >> $GITHUB_ENV | |
sudo mkdir ${MINIO_DIR} | |
sudo docker run -d --net=container:${TEST_CONTAINER_NAME} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v ${MINIO_DIR}:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005" | |
sleep 5s | |
if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then | |
echo "Minio container is not running" | |
exit 1 | |
fi | |
- name: Start infinity release version | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
cmake-build-release/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > minio_parallel_test_release.log 2> minio_parallel_test_release_error.log" & | |
- name: Run minio parallel test | |
if: ${{ !cancelled() && !failure() }} | |
id: run_minio_parallel_test_release | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "cd /infinity/ && | |
python3 tools/run_parallel_test.py" | |
sleep 1s | |
- name: Collect infinity log | |
if: ${{ !cancelled() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c \ | |
"mv /infinity/minio_parallel_test_release.log /infinity_log/minio_parallel_test_release.log && | |
mv /infinity/minio_parallel_test_release_error.log /infinity_log/minio_parallel_test_release_error.log && | |
echo 'minio parallel test release log' && | |
tail -n 1000 /infinity_log/minio_parallel_test_release.log && | |
echo 'minio parallel test release error log' && | |
tail -n 1000 /infinity_log//minio_parallel_test_release_error.log" | |
- name: Remove test container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${TEST_CONTAINER_NAME} | |
sudo rm -rf /var/infinity/${TEST_CONTAINER_NAME}/ | |
- name: Remove minio container | |
if: always() | |
run: | | |
sudo docker rm -f -v ${MINIO_CONTAINER} | |
release_benchmark: | |
name: release_benchmark | |
if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
runs-on: [ "self-hosted", "benchmark" ] | |
needs: release_configure_and_build | |
outputs: | |
BUILD_CONTAINER_NAME: ${{ steps.start.outputs.BUILD_CONTAINER_NAME }} | |
TEST_CONTAINER_NAME: ${{ steps.start.outputs.TEST_CONTAINER_NAME }} | |
env: | |
BUILD_IP: ${{needs.release_configure_and_build.outputs.BUILD_IP}} | |
CONTAINER_NAME: ${{needs.release_configure_and_build.outputs.CONTAINER_NAME}} | |
steps: | |
- name: Show PR labels | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
echo "Workflow triggered by ${{ github.event_name }}" | |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
fi | |
- name: Ensure workspace ownership | |
if: ${{ !cancelled() && !failure() }} | |
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
- name: Check out code | |
if: ${{ !cancelled() && !failure() }} | |
uses: actions/checkout@v5 | |
with: | |
fetch-depth: 0 | |
fetch-tags: false | |
- name: Start test container | |
id: start | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
TEST_CONTAINER_NAME=infinity_test_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
echo "TEST_CONTAINER_NAME=${TEST_CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "BUILD_CONTAINER_NAME=${CONTAINER_NAME}" >> $GITHUB_ENV | |
echo "USERNAME=${USERNAME}" >> $GITHUB_ENV | |
echo "PASSWORD=${PASSWORD}" >> $GITHUB_ENV | |
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${TEST_CONTAINER_NAME} -e TZ=$TZ -v $PWD:/infinity infiniflow/infinity_builder:ubuntu20_clang20 | |
- name: Download artifacts | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: CONTAINER_NAME, BUILD_IP | |
script: | | |
mkdir -p /home/infiniflow/artifacts/infinity/${CONTAINER_NAME} | |
rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/infinity /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/ | |
rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/knn_query_benchmark /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/ | |
rsync -a infiniflow@${BUILD_IP}:/home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/knn_import_benchmark /home/infiniflow/artifacts/infinity/${CONTAINER_NAME}/ | |
- name: SSH to host server | |
id: ssh | |
uses: appleboy/[email protected] | |
if: ${{ !cancelled() && !failure() }} | |
with: | |
host: 172.17.0.1 | |
username: ${{ env.USERNAME }} | |
password: ${{ env.PASSWORD }} | |
envs: TEST_CONTAINER_NAME, BUILD_CONTAINER_NAME, RUNNER_NAME, RUNNER_WORKSPACE_PREFIX | |
script: | | |
sudo docker exec ${TEST_CONTAINER_NAME} mkdir -p /infinity/cmake-build-release/benchmark/local_infinity/ | |
sudo cp /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/knn_query_benchmark ${RUNNER_WORKSPACE_PREFIX}/${RUNNER_NAME}/infinity/infinity/cmake-build-release/benchmark/local_infinity/ | |
sudo cp /home/infiniflow/artifacts/infinity/${BUILD_CONTAINER_NAME}/knn_import_benchmark ${RUNNER_WORKSPACE_PREFIX}/${RUNNER_NAME}/infinity/infinity/cmake-build-release/benchmark/local_infinity/ | |
- name: Install pysdk | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
sudo docker exec ${TEST_CONTAINER_NAME} bash -c "rm -rf /root/.config/pip/pip.conf && | |
cd /infinity/ && | |
pip3 uninstall -y infinity-sdk && | |
pip3 install python/infinity_sdk -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
- name: Prepare sift dataset | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME} | |
echo "RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX}" >> $GITHUB_ENV | |
sudo chmod +x ./tools/ci_tools/check_benchmark_result.py | |
sudo mkdir -p test/data/benchmark | |
sudo ln -s ${RUNNER_WORKSPACE_PREFIX}/benchmark/sift1M test/data/benchmark/sift_1m | |
- name: Golden benchmark sift | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
if [[ -f "${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_1_thread.log" ]]; then | |
echo "golden benchmark result already exists!" | |
exit 0 | |
fi | |
rm -fr $PWD/db_tmp && mkdir -p $PWD/db_tmp && cat ${RUNNER_WORKSPACE_PREFIX}/benchmark/infinity_conf.toml | sed -e "s|/var/infinity|$PWD/db_tmp|g" > $PWD/db_tmp/infinity_conf.toml | |
${RUNNER_WORKSPACE_PREFIX}/benchmark/knn_import_benchmark --dataset=sift --dataset_dir=$PWD/test/data --infinity_dir=$PWD/db_tmp | |
echo "1 50" | ${RUNNER_WORKSPACE_PREFIX}/benchmark/knn_query_benchmark sift 200 false $PWD/test/data $PWD/db_tmp | sudo tee ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_1_thread.log | |
echo "8 50" | ${RUNNER_WORKSPACE_PREFIX}/benchmark/knn_query_benchmark sift 200 false $PWD/test/data $PWD/db_tmp | sudo tee ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_8_threads.log | |
- name: Latest benchmark sift | |
if: ${{ !cancelled() && !failure() }} | |
run: | | |
rm -fr $PWD/db_tmp && mkdir -p $PWD/db_tmp && cat conf/infinity_conf.toml | sed -e "s|/var/infinity|$PWD/db_tmp|g" > $PWD/db_tmp/infinity_conf.toml | |
./cmake-build-release/benchmark/local_infinity/knn_import_benchmark --dataset=sift --dataset_dir=$PWD/test/data --infinity_dir=$PWD/db_tmp --config_path=db_tmp/infinity_conf.toml | |
echo "1 50" | ./cmake-build-release/benchmark/local_infinity/knn_query_benchmark sift 200 false $PWD/test/data $PWD/db_tmp db_tmp/infinity_conf.toml | tee benchmark_sift_1_thread.log | |
echo "8 50" | ./cmake-build-release/benchmark/local_infinity/knn_query_benchmark sift 200 false $PWD/test/data $PWD/db_tmp db_tmp/infinity_conf.toml | tee benchmark_sift_8_threads.log | |
./tools/ci_tools/check_benchmark_result.py ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_1_thread.log benchmark_sift_1_thread.log | |
./tools/ci_tools/check_benchmark_result.py ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_8_threads.log benchmark_sift_8_threads.log | |
- name: Remove test container | |
if: always() | |
run: sudo docker rm -f -v ${TEST_CONTAINER_NAME} |