From 3cda89a683bcc6d124e55241a2f8100933a231ed Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Mon, 2 Dec 2024 11:11:33 +0100 Subject: [PATCH] Test github actions for Linux (#17094) * add github actions * output info * add user * revert user * use conan-tests for docker testing * get docker id * revert * try changes * update cache and checkout * revert changes * do not use spaces * fix * wip * wip * wip * revert changes * Revert "revert changes" This reverts commit 58d175d9772fa0b86e2c3db5d08ae71d9af6f116. * Revert "wip" This reverts commit 597bc0074946e78e53e696ab2f4988abe4041bf4. * try without image * remove ninja mark * wip * remove unused * wip * minot changes * fail if all tests are skipped * minor changes * reduce py versions * fix python * does not need container * revert change * add name * change name * use v5 * Update .github/workflows/linux-tests.yml * Update .github/workflows/linux-tests.yml * recover bump_dev_version * add deply to pypitest * remove fish * add python 3.13 --- .ci/docker/conan-tests | 146 +++++++++++++++ .github/workflows/linux-tests.yml | 177 ++++++++++++++++++ pytest.ini | 1 + .../functional/command/dockerfiles/Dockerfile | 2 +- test/functional/command/runner_test.py | 11 +- 5 files changed, 335 insertions(+), 2 deletions(-) create mode 100644 .ci/docker/conan-tests create mode 100644 .github/workflows/linux-tests.yml diff --git a/.ci/docker/conan-tests b/.ci/docker/conan-tests new file mode 100644 index 00000000000..2d176b181fe --- /dev/null +++ b/.ci/docker/conan-tests @@ -0,0 +1,146 @@ +FROM ubuntu:24.04 + +LABEL maintainer="Conan.io " + +ENV DEBIAN_FRONTEND=noninteractive + +ENV PY36=3.6.15 \ + PY38=3.8.6 \ + PY39=3.9.2 \ + PY312=3.12.3 \ + PY313=3.13.0 \ + CMAKE_3_15_7=/usr/share/cmake-3.15.7/bin/cmake \ + CMAKE_3_16_9=/usr/share/cmake-3.16.9/bin/cmake \ + CMAKE_3_17_5=/usr/share/cmake-3.17.5/bin/cmake \ + CMAKE_3_19_7=/usr/share/cmake-3.19.7/bin/cmake \ + CMAKE_3_23_5=/usr/share/cmake-3.23.5/bin/cmake \ + GCC_9=/usr/bin/gcc-9 \ + GXX_9=/usr/bin/g++-9 \ + GCC_11=/usr/bin/gcc-11 \ + GXX_11=/usr/bin/g++-11 \ + CLANG_14=/usr/bin/clang-14 \ + CLANGXX_14=/usr/bin/clang++-14 \ + BAZEL_6_3_2=6.3.2 \ + BAZEL_7_1_2=7.1.2 + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + software-properties-common \ + build-essential \ + libtool \ + automake \ + autoconf \ + pkg-config \ + gettext \ + git \ + curl \ + make \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + wget \ + llvm \ + libncurses5-dev \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + libffi-dev \ + liblzma-dev \ + python3-openssl \ + ca-certificates \ + sudo \ + tar \ + linux-libc-dev \ + subversion \ + subversion-tools \ + ninja-build \ + gcc-9 \ + g++-9 \ + gcc-11 \ + g++-11 \ + clang-14 \ + clang++-14 \ + gcc-arm-linux-gnueabihf \ + g++-arm-linux-gnueabihf \ + unzip \ + apt-transport-https \ + gnupg-agent \ + gcc-9-multilib \ + g++-9-multilib \ + gcc-11-multilib \ + g++-11-multilib \ + scons && \ + # fix: asm/errno.h: No such file or directory + ln -s /usr/include/asm-generic/ /usr/include/asm && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + apt-get update && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu jammy stable" && \ + apt-get update && \ + apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io && \ + rm -rf /var/lib/apt/lists/* + +RUN useradd -m -s /bin/bash conan && \ + echo 'conan ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +ENV HOME /home/conan +ENV PYENV_ROOT $HOME/.pyenv +ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:/usr/bin:/bin:$PATH + +RUN curl https://pyenv.run | bash && \ + pyenv install $PY36 && \ + pyenv install $PY38 && \ + pyenv install $PY39 && \ + pyenv install $PY312 && \ + pyenv install $PY313 && \ + pyenv global $PY39 && \ + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python get-pip.py && \ + rm get-pip.py + +RUN chown -R conan:conan $HOME + +USER root + +RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.7/cmake-3.15.7-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.15.7-Linux-x86_64.tar.gz && mv cmake-3.15.7-Linux-x86_64 /usr/share/cmake-3.15.7 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.16.9/cmake-3.16.9-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.16.9-Linux-x86_64.tar.gz && mv cmake-3.16.9-Linux-x86_64 /usr/share/cmake-3.16.9 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.17.5-Linux-x86_64.tar.gz && mv cmake-3.17.5-Linux-x86_64 /usr/share/cmake-3.17.5 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.19.7/cmake-3.19.7-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.19.7-Linux-x86_64.tar.gz && mv cmake-3.19.7-Linux-x86_64 /usr/share/cmake-3.19.7 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.23.5/cmake-3.23.5-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.23.5-Linux-x86_64.tar.gz && mv cmake-3.23.5-linux-x86_64/ /usr/share/cmake-3.23.5 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_15_7 10 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_16_9 20 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_17_5 30 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_19_7 40 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_23_5 50 + +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 10 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 30 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 10 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 30 && \ + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 10 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 10 && \ + update-alternatives --set gcc /usr/bin/gcc-9 && \ + update-alternatives --set g++ /usr/bin/g++-9 && \ + update-alternatives --set clang /usr/bin/clang-14 && \ + update-alternatives --set clang++ /usr/bin/clang++-14 + +RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \ + wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_6_3_2}/bazel-${BAZEL_6_3_2}-linux-x86_64 && \ + chmod +x bazel-${BAZEL_6_3_2}-linux-x86_64 && \ + mv bazel-${BAZEL_6_3_2}-linux-x86_64 /usr/share/bazel-$BAZEL_6_3_2/bin/bazel && \ + mkdir -p /usr/share/bazel-$BAZEL_7_1_2/bin && \ + wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_7_1_2}/bazel-${BAZEL_7_1_2}-linux-x86_64 && \ + chmod +x bazel-${BAZEL_7_1_2}-linux-x86_64 && \ + mv bazel-${BAZEL_7_1_2}-linux-x86_64 /usr/share/bazel-$BAZEL_7_1_2/bin/bazel + +USER conan +WORKDIR $HOME + +CMD ["/bin/bash"] diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml new file mode 100644 index 00000000000..fb229df9798 --- /dev/null +++ b/.github/workflows/linux-tests.yml @@ -0,0 +1,177 @@ +name: Linux tests + +on: + push: + branches: + - develop2 + - release/* + pull_request: + branches: + - '*' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build_container: + runs-on: ubuntu-latest + name: Build and Publish Docker Image for testing Conan + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Calculate Dockerfile checksum + id: dockerfile_hash + run: | + DOCKERFILE_HASH=$(find ./.ci/docker/conan-tests -type f -exec sha256sum {} \; | sha256sum | cut -d' ' -f1) + echo "hash=$DOCKERFILE_HASH" >> $GITHUB_OUTPUT + + - name: Cache Docker image + id: cache_docker_image + uses: actions/cache@v4 + with: + path: docker_image_cache + key: ${{ runner.os }}-docker-${{ steps.dockerfile_hash.outputs.hash }} + + - name: Load Docker image from cache + if: steps.cache_docker_image.outputs.cache-hit == 'true' + run: docker load -i docker_image_cache/conan-tests.tar + + - name: Build Docker image (if not cached) + if: steps.cache_docker_image.outputs.cache-hit != 'true' + run: | + docker build \ + -t ghcr.io/${{ github.repository_owner }}/conan-tests:latest \ + -f ./.ci/docker/conan-tests . + mkdir -p docker_image_cache + docker save ghcr.io/${{ github.repository_owner }}/conan-tests:latest -o docker_image_cache/conan-tests.tar + + - name: Push Docker image to GHCR + if: steps.cache_docker_image.outputs.cache-hit != 'true' + run: docker push ghcr.io/${{ github.repository_owner }}/conan-tests:latest + + linux_test_suite: + needs: build_container + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest + options: --user conan + strategy: + matrix: + python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] + test-type: ['unittests', 'integration', 'functional'] + name: Conan ${{ matrix.test-type }} (${{ matrix.python-version }}) + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + run: | + pyenv global ${{ matrix.python-version }} + python --version + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }} + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -r conans/requirements.txt + pip install -r conans/requirements_dev.txt + pip install -r conans/requirements_server.txt + pip install meson + + - name: Run tests + shell: bash + run: | + if [ "${{ matrix.test-type }}" == "unittests" ]; then + pytest test/unittests --durations=20 -n 4 + elif [ "${{ matrix.test-type }}" == "integration" ]; then + pytest test/integration --durations=20 -n 4 + elif [ "${{ matrix.test-type }}" == "functional" ]; then + pytest test/functional --durations=20 -n 4 + fi + + linux_docker_tests: + runs-on: ubuntu-latest + name: Docker Runner Tests + strategy: + matrix: + python-version: [3.12, 3.9] + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -r conans/requirements.txt + pip install -r conans/requirements_dev.txt + pip install -r conans/requirements_server.txt + pip install -r conans/requirements_runner.txt + + - name: Run tests + shell: bash + run: | + pytest -m docker_runner --durations=20 -rs + + deploy_to_pypi_test: + needs: [build_container, linux_test_suite, linux_docker_tests] + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/develop2' + name: Deploy to TestPyPI + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install twine + + - name: Bump Dev Version + run: | + python .ci/bump_dev_version.py + + - name: Build Package + run: | + python setup.py sdist + + - name: Upload to TestPyPI + env: + TWINE_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }} + run: | + python -m twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/* + + - name: Deploy conan-server to TestPyPI + env: + TWINE_USERNAME: ${{ secrets.TEST_PYPI_SERVER_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_SERVER_PASSWORD }} + run: | + rm -rf dist/ + mv setup_server.py setup.py + python setup.py sdist + python -m twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/* diff --git a/pytest.ini b/pytest.ini index a69be4e9060..26456fc871e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,4 @@ [pytest] norecursedirs = '.*', 'dist', 'CVS', '_darcs', '{arch}', '*.egg', 'venv', 'assets' testpaths = 'test' +markers = docker_runner: Mark tests that require Docker to run. diff --git a/test/functional/command/dockerfiles/Dockerfile b/test/functional/command/dockerfiles/Dockerfile index 068de9a2efd..a90abd2b9cf 100644 --- a/test/functional/command/dockerfiles/Dockerfile +++ b/test/functional/command/dockerfiles/Dockerfile @@ -8,4 +8,4 @@ RUN apt-get update \ python3-venv \ && rm -rf /var/lib/apt/lists/* COPY . /root/conan-io -RUN cd /root/conan-io && pip install -e . \ No newline at end of file +RUN cd /root/conan-io && pip install -e . diff --git a/test/functional/command/runner_test.py b/test/functional/command/runner_test.py index fb601288aeb..b385873c74a 100644 --- a/test/functional/command/runner_test.py +++ b/test/functional/command/runner_test.py @@ -36,6 +36,7 @@ def dockerfile_path(name=None): return path +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_cache_shared(): """ @@ -79,6 +80,7 @@ def test_create_docker_runner_cache_shared(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_cache_shared_profile_from_cache(): """ @@ -122,6 +124,7 @@ def test_create_docker_runner_cache_shared_profile_from_cache(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_cache_shared_profile_folder(): """ @@ -165,6 +168,7 @@ def test_create_docker_runner_cache_shared_profile_folder(): assert "[100%] Built target example" in client.out assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_dockerfile_folder_path(): """ @@ -235,6 +239,7 @@ def test_create_docker_runner_dockerfile_folder_path(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_profile_default_folder(): """ @@ -280,6 +285,7 @@ def test_create_docker_runner_profile_default_folder(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_dockerfile_file_path(): """ @@ -324,9 +330,9 @@ def test_create_docker_runner_dockerfile_file_path(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") @pytest.mark.parametrize("build_type,shared", [("Release", False), ("Debug", True)]) -@pytest.mark.tool("ninja") def test_create_docker_runner_with_ninja(build_type, shared): conanfile = textwrap.dedent(""" import os @@ -389,6 +395,7 @@ def package(self): assert 'cmake -G "Ninja"' in client.out assert "main: {}!".format(build_type) in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_from_configfile(): """ @@ -443,6 +450,7 @@ def test_create_docker_runner_from_configfile(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_from_configfile_with_args(): """ @@ -499,6 +507,7 @@ def test_create_docker_runner_from_configfile_with_args(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_default_build_profile(): """