From 0f2de7d0a275028cd9519a5ae22f0f97af1ffa2f Mon Sep 17 00:00:00 2001 From: Zakelly Date: Sun, 21 Apr 2024 11:11:07 +0800 Subject: [PATCH] Enable CI for kvrocks2redis (#2175) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 纪华裕 Co-authored-by: Twice Co-authored-by: hulk --- .github/workflows/kvrocks.yaml | 83 ++++++++++++++++--- utils/kvrocks2redis/tests/populate-kvrocks.py | 11 ++- 2 files changed, 80 insertions(+), 14 deletions(-) diff --git a/.github/workflows/kvrocks.yaml b/.github/workflows/kvrocks.yaml index f28541434f8..b7767575574 100644 --- a/.github/workflows/kvrocks.yaml +++ b/.github/workflows/kvrocks.yaml @@ -223,13 +223,21 @@ jobs: path: | ~/local/bin/redis-cli key: ${{ runner.os }}-${{ runner.arch }}-redis-cli + - name: Cache redis server + id: cache-redis-server + uses: actions/cache@v4 + with: + path: | + ~/local/bin/redis-server + key: ${{ runner.os }}-${{ runner.arch }}-redis-server - name: Install redis - if: steps.cache-redis.outputs.cache-hit != 'true' + if: ${{ steps.cache-redis.outputs.cache-hit != 'true' || steps.cache-redis-server.outputs.cache-hit != 'true' }} run: | - curl -O https://download.redis.io/releases/redis-6.2.7.tar.gz - tar -xzvf redis-6.2.7.tar.gz + curl -O https://download.redis.io/releases/redis-6.2.14.tar.gz + tar -xzvf redis-6.2.14.tar.gz mkdir -p $HOME/local/bin - pushd redis-6.2.7 && BUILD_TLS=yes make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd + pushd redis-6.2.14 && BUILD_TLS=yes make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd + pushd redis-6.2.14 && BUILD_TLS=yes make -j$NPROC redis-server && mv src/redis-server $HOME/local/bin/ && popd - uses: actions/checkout@v4 with: @@ -256,7 +264,7 @@ jobs: ./x.py build -j$NPROC --unittest --compiler ${{ matrix.compiler }} ${{ matrix.without_jemalloc }} \ ${{ matrix.without_luajit }} ${{ matrix.with_ninja }} ${{ matrix.with_sanitizer }} ${{ matrix.with_openssl }} \ ${{ matrix.new_encoding }} ${{ matrix.with_speedb }} ${{ env.CMAKE_EXTRA_DEFS }} - + - name: Build Kvrocks (SonarCloud) if: ${{ matrix.sonarcloud }} run: | @@ -293,6 +301,30 @@ jobs: fi ./x.py test go build $GOCASE_RUN_ARGS ${{ matrix.ignore_when_tsan}} + - name: Install redis-py + run: pip3 install redis==4.3.6 + + - name: Run kvrocks2redis Test + # Currently, when enabling Tsan/Asan or running in macOS 11/14, the value mismatch in destination redis server. + # See https://github.com/apache/kvrocks/issues/2195. + if: ${{ !contains(matrix.name, 'Tsan') && !contains(matrix.name, 'Asan') && !startsWith(matrix.os, 'macos') }} + run: | + ulimit -c unlimited + export LSAN_OPTIONS="suppressions=$(realpath ./tests/lsan-suppressions)" + export TSAN_OPTIONS="suppressions=$(realpath ./tests/tsan-suppressions)" + $HOME/local/bin/redis-server --daemonize yes + mkdir -p kvrocks2redis-ci-data + ./build/kvrocks --dir `pwd`/kvrocks2redis-ci-data --pidfile `pwd`/kvrocks.pid --daemonize yes + sleep 10s + echo -en "data-dir `pwd`/kvrocks2redis-ci-data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf + cat ./kvrocks2redis-ci.conf + ./build/kvrocks2redis -c ./kvrocks2redis-ci.conf + sleep 10s + python3 utils/kvrocks2redis/tests/populate-kvrocks.py --password="" --flushdb=true + sleep 10s + ps aux + python3 utils/kvrocks2redis/tests/check_consistency.py --src_password="" + - name: Find reports and crashes if: always() run: | @@ -397,7 +429,7 @@ jobs: if: ${{ startsWith(matrix.image, 'centos') }} run: | yum install -y centos-release-scl-rh - yum install -y devtoolset-11 python3 autoconf automake wget git gcc gcc-c++ + yum install -y devtoolset-11 python3 python3-pip autoconf automake wget git gcc gcc-c++ echo "NPROC=$(nproc)" >> $GITHUB_ENV mv /usr/bin/gcc /usr/bin/gcc-4.8.5 ln -s /opt/rh/devtoolset-11/root/bin/gcc /usr/bin/gcc @@ -408,13 +440,13 @@ jobs: if: ${{ startsWith(matrix.image, 'archlinux') }} run: | pacman -Syu --noconfirm - pacman -Sy --noconfirm autoconf automake python3 git wget which cmake make gcc + pacman -Sy --noconfirm autoconf automake python3 python-redis git wget which cmake make gcc echo "NPROC=$(nproc)" >> $GITHUB_ENV - name: Setup openSUSE if: ${{ startsWith(matrix.image, 'opensuse') }} run: | - zypper install -y gcc11 gcc11-c++ make wget git autoconf automake python3 curl tar gzip cmake go + zypper install -y gcc11 gcc11-c++ make wget git autoconf automake python3 python3-pip curl tar gzip cmake go update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-11 100 update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-11 100 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 @@ -429,13 +461,22 @@ jobs: ~/local/bin/redis-cli key: ${{ matrix.image }}-redis-cli + - name: Cache redis server + id: cache-redis-server + uses: actions/cache@v3 + with: + path: | + ~/local/bin/redis-server + key: ${{ matrix.image }}-redis-server + - name: Install redis - if: steps.cache-redis.outputs.cache-hit != 'true' + if: ${{ steps.cache-redis.outputs.cache-hit != 'true' || steps.cache-redis-server.outputs.cache-hit != 'true' }} run: | - curl -O https://download.redis.io/releases/redis-6.2.7.tar.gz - tar -xzvf redis-6.2.7.tar.gz + curl -O https://download.redis.io/releases/redis-6.2.14.tar.gz + tar -xzvf redis-6.2.14.tar.gz mkdir -p $HOME/local/bin - pushd redis-6.2.7 && USE_JEMALLOC=no make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd + pushd redis-6.2.14 && USE_JEMALLOC=no make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd + pushd redis-6.2.14 && USE_JEMALLOC=no make -j$NPROC redis-server && mv src/redis-server $HOME/local/bin/ && popd - name: Install cmake if: ${{ startsWith(matrix.image, 'centos') }} @@ -465,6 +506,24 @@ jobs: GOCASE_RUN_ARGS="" ./x.py test go build $GOCASE_RUN_ARGS + - name: Install redis-py + if: ${{ !startsWith(matrix.image, 'archlinux') }} # already installed + run: pip3 install redis==4.3.6 + + - name: Run kvrocks2redis Test + run: | + $HOME/local/bin/redis-server --daemonize yes + mkdir -p kvrocks2redis-ci-data + ./build/kvrocks --dir `pwd`/kvrocks2redis-ci-data --pidfile `pwd`/kvrocks.pid --daemonize yes + sleep 10s + echo -en "data-dir `pwd`/kvrocks2redis-ci-data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf + cat ./kvrocks2redis-ci.conf + ./build/kvrocks2redis -c ./kvrocks2redis-ci.conf + sleep 10s + python3 utils/kvrocks2redis/tests/populate-kvrocks.py --password="" --flushdb=true + sleep 10s + python3 utils/kvrocks2redis/tests/check_consistency.py --src_password="" + required: if: always() name: Required diff --git a/utils/kvrocks2redis/tests/populate-kvrocks.py b/utils/kvrocks2redis/tests/populate-kvrocks.py index 8883a0dfd36..fefae86a0fe 100644 --- a/utils/kvrocks2redis/tests/populate-kvrocks.py +++ b/utils/kvrocks2redis/tests/populate-kvrocks.py @@ -146,8 +146,10 @@ def run_test(client, cases : list): print('******* Some case test fail *******') for cmd in fails: print(cmd) + return False else: print('All case passed.') + return True if __name__ == '__main__': @@ -155,5 +157,10 @@ def run_test(client, cases : list): client = redis.Redis(host=args.host, port=args.port, decode_responses=True, password=args.password) if args.flushdb: client.flushdb() - run_test(client, PopulateCases) - run_test(client, AppendCases) + succ = True + if not run_test(client, PopulateCases): + succ = False + if not run_test(client, AppendCases): + succ = False + if not succ: + raise AssertionError("Test failed. See details above.")