Test github actions for Linux (#17094)

* add github actions * output info * add user * revert user * use conan-tests for docker testing * get docker id * revert * try changes * update cache and checkout * revert changes * do not use spaces * fix * wip * wip * wip * revert changes * Revert "revert changes" This reverts commit 58d175d. * Revert "wip" This reverts commit 597bc00. * try without image * remove ninja mark * wip * remove unused * wip * minot changes * fail if all tests are skipped * minor changes * reduce py versions * fix python * does not need container * revert change * add name * change name * use v5 * Update .github/workflows/linux-tests.yml * Update .github/workflows/linux-tests.yml * recover bump_dev_version * add deply to pypitest * remove fish * add python 3.13
conan-io · Dec 2, 2024 · 3cda89a · 3cda89a
1 parent 1cd1b23
commit 3cda89a
Show file tree

Hide file tree

Showing 5 changed files with 335 additions and 2 deletions.
diff --git a/.ci/docker/conan-tests b/.ci/docker/conan-tests
@@ -0,0 +1,146 @@
+FROM ubuntu:24.04
+
+LABEL maintainer="Conan.io <[email protected]>"
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+ENV PY36=3.6.15 \
+    PY38=3.8.6 \
+    PY39=3.9.2 \
+    PY312=3.12.3 \
+    PY313=3.13.0 \
+    CMAKE_3_15_7=/usr/share/cmake-3.15.7/bin/cmake \
+    CMAKE_3_16_9=/usr/share/cmake-3.16.9/bin/cmake \
+    CMAKE_3_17_5=/usr/share/cmake-3.17.5/bin/cmake \
+    CMAKE_3_19_7=/usr/share/cmake-3.19.7/bin/cmake \
+    CMAKE_3_23_5=/usr/share/cmake-3.23.5/bin/cmake \
+    GCC_9=/usr/bin/gcc-9 \
+    GXX_9=/usr/bin/g++-9 \
+    GCC_11=/usr/bin/gcc-11 \
+    GXX_11=/usr/bin/g++-11 \
+    CLANG_14=/usr/bin/clang-14 \
+    CLANGXX_14=/usr/bin/clang++-14 \
+    BAZEL_6_3_2=6.3.2 \
+    BAZEL_7_1_2=7.1.2
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        software-properties-common \
+        build-essential \
+        libtool \
+        automake \
+        autoconf \
+        pkg-config \
+        gettext \
+        git \
+        curl \
+        make \
+        libssl-dev \
+        zlib1g-dev \
+        libbz2-dev \
+        libreadline-dev \
+        libsqlite3-dev \
+        wget \
+        llvm \
+        libncurses5-dev \
+        libncursesw5-dev \
+        xz-utils \
+        tk-dev \
+        libffi-dev \
+        liblzma-dev \
+        python3-openssl \
+        ca-certificates \
+        sudo \
+        tar \
+        linux-libc-dev \
+        subversion \
+        subversion-tools \
+        ninja-build \
+        gcc-9 \
+        g++-9 \
+        gcc-11 \
+        g++-11 \
+        clang-14 \
+        clang++-14 \
+        gcc-arm-linux-gnueabihf \
+        g++-arm-linux-gnueabihf \
+        unzip \
+        apt-transport-https \
+        gnupg-agent \
+        gcc-9-multilib \
+        g++-9-multilib \
+        gcc-11-multilib \
+        g++-11-multilib \
+        scons && \
+    # fix: asm/errno.h: No such file or directory
+    ln -s /usr/include/asm-generic/ /usr/include/asm && \
+    add-apt-repository -y ppa:ubuntu-toolchain-r/test && \
+    apt-get update && \
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
+    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu jammy stable" && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN useradd -m -s /bin/bash conan && \
+    echo 'conan ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+ENV HOME /home/conan
+ENV PYENV_ROOT $HOME/.pyenv
+ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:/usr/bin:/bin:$PATH
+
+RUN curl https://pyenv.run | bash && \
+    pyenv install $PY36 && \
+    pyenv install $PY38 && \
+    pyenv install $PY39 && \
+    pyenv install $PY312 && \
+    pyenv install $PY313 && \
+    pyenv global $PY39 && \
+    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python get-pip.py && \
+    rm get-pip.py
+
+RUN chown -R conan:conan $HOME
+
+USER root
+
+RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.7/cmake-3.15.7-Linux-x86_64.tar.gz && \
+    tar -xvzf cmake-3.15.7-Linux-x86_64.tar.gz && mv cmake-3.15.7-Linux-x86_64 /usr/share/cmake-3.15.7  && \
+    wget https://github.com/Kitware/CMake/releases/download/v3.16.9/cmake-3.16.9-Linux-x86_64.tar.gz   && \
+    tar -xvzf cmake-3.16.9-Linux-x86_64.tar.gz && mv cmake-3.16.9-Linux-x86_64 /usr/share/cmake-3.16.9 && \
+    wget https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-Linux-x86_64.tar.gz   && \
+    tar -xvzf cmake-3.17.5-Linux-x86_64.tar.gz && mv cmake-3.17.5-Linux-x86_64 /usr/share/cmake-3.17.5 && \
+    wget https://github.com/Kitware/CMake/releases/download/v3.19.7/cmake-3.19.7-Linux-x86_64.tar.gz   && \
+    tar -xvzf cmake-3.19.7-Linux-x86_64.tar.gz && mv cmake-3.19.7-Linux-x86_64 /usr/share/cmake-3.19.7 && \
+    wget https://github.com/Kitware/CMake/releases/download/v3.23.5/cmake-3.23.5-Linux-x86_64.tar.gz   && \
+    tar -xvzf cmake-3.23.5-Linux-x86_64.tar.gz && mv cmake-3.23.5-linux-x86_64/ /usr/share/cmake-3.23.5 && \
+    update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_15_7 10 && \
+    update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_16_9 20 && \
+    update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_17_5 30 && \
+    update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_19_7 40 && \
+    update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_23_5 50
+
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 10 && \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 30 && \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 10 && \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 30 && \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 10 && \
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 10 && \
+    update-alternatives --set gcc /usr/bin/gcc-9 && \
+    update-alternatives --set g++ /usr/bin/g++-9 && \
+    update-alternatives --set clang /usr/bin/clang-14 && \
+    update-alternatives --set clang++ /usr/bin/clang++-14
+
+RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \
+    wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_6_3_2}/bazel-${BAZEL_6_3_2}-linux-x86_64 && \
+    chmod +x bazel-${BAZEL_6_3_2}-linux-x86_64 && \
+    mv bazel-${BAZEL_6_3_2}-linux-x86_64 /usr/share/bazel-$BAZEL_6_3_2/bin/bazel && \
+    mkdir -p /usr/share/bazel-$BAZEL_7_1_2/bin && \
+    wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_7_1_2}/bazel-${BAZEL_7_1_2}-linux-x86_64 && \
+    chmod +x bazel-${BAZEL_7_1_2}-linux-x86_64 && \
+    mv bazel-${BAZEL_7_1_2}-linux-x86_64 /usr/share/bazel-$BAZEL_7_1_2/bin/bazel
+
+USER conan
+WORKDIR $HOME
+
+CMD ["/bin/bash"]
diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml
@@ -0,0 +1,177 @@
+name: Linux tests
+
+on:
+  push:
+    branches:
+      - develop2
+      - release/*
+  pull_request:
+    branches:
+      - '*'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build_container:
+    runs-on: ubuntu-latest
+    name: Build and Publish Docker Image for testing Conan
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Calculate Dockerfile checksum
+        id: dockerfile_hash
+        run: |
+          DOCKERFILE_HASH=$(find ./.ci/docker/conan-tests -type f -exec sha256sum {} \; | sha256sum | cut -d' ' -f1)
+          echo "hash=$DOCKERFILE_HASH" >> $GITHUB_OUTPUT
+
+      - name: Cache Docker image
+        id: cache_docker_image
+        uses: actions/cache@v4
+        with:
+          path: docker_image_cache
+          key: ${{ runner.os }}-docker-${{ steps.dockerfile_hash.outputs.hash }}
+
+      - name: Load Docker image from cache
+        if: steps.cache_docker_image.outputs.cache-hit == 'true'
+        run: docker load -i docker_image_cache/conan-tests.tar
+
+      - name: Build Docker image (if not cached)
+        if: steps.cache_docker_image.outputs.cache-hit != 'true'
+        run: |
+          docker build \
+          -t ghcr.io/${{ github.repository_owner }}/conan-tests:latest \
+          -f ./.ci/docker/conan-tests .
+          mkdir -p docker_image_cache
+          docker save ghcr.io/${{ github.repository_owner }}/conan-tests:latest -o docker_image_cache/conan-tests.tar
+
+      - name: Push Docker image to GHCR
+        if: steps.cache_docker_image.outputs.cache-hit != 'true'
+        run: docker push ghcr.io/${{ github.repository_owner }}/conan-tests:latest
+
+  linux_test_suite:
+    needs: build_container
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest
+      options: --user conan
+    strategy:
+      matrix:
+        python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15']
+        test-type: ['unittests', 'integration', 'functional']
+    name: Conan ${{ matrix.test-type }} (${{ matrix.python-version }}) 
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: |
+          pyenv global ${{ matrix.python-version }}
+          python --version
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }}
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -r conans/requirements.txt
+          pip install -r conans/requirements_dev.txt
+          pip install -r conans/requirements_server.txt
+          pip install meson
+
+      - name: Run tests
+        shell: bash
+        run: |
+          if [ "${{ matrix.test-type }}" == "unittests" ]; then
+            pytest test/unittests --durations=20 -n 4
+          elif [ "${{ matrix.test-type }}" == "integration" ]; then
+            pytest test/integration --durations=20 -n 4
+          elif [ "${{ matrix.test-type }}" == "functional" ]; then
+            pytest test/functional --durations=20 -n 4
+          fi
+
+  linux_docker_tests:
+    runs-on: ubuntu-latest
+    name: Docker Runner Tests
+    strategy:
+      matrix:
+        python-version: [3.12, 3.9]
+    steps:
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -r conans/requirements.txt
+          pip install -r conans/requirements_dev.txt
+          pip install -r conans/requirements_server.txt
+          pip install -r conans/requirements_runner.txt
+
+      - name: Run tests
+        shell: bash
+        run: |
+          pytest -m docker_runner --durations=20 -rs
+
+  deploy_to_pypi_test:
+    needs: [build_container, linux_test_suite, linux_docker_tests]
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/develop2'
+    name: Deploy to TestPyPI
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install twine
+
+      - name: Bump Dev Version
+        run: |
+          python .ci/bump_dev_version.py
+
+      - name: Build Package
+        run: |
+          python setup.py sdist
+
+      - name: Upload to TestPyPI
+        env:
+          TWINE_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }}
+        run: |
+          python -m twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/*
+
+      - name: Deploy conan-server to TestPyPI
+        env:
+          TWINE_USERNAME: ${{ secrets.TEST_PYPI_SERVER_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_SERVER_PASSWORD }}
+        run: |
+          rm -rf dist/
+          mv setup_server.py setup.py
+          python setup.py sdist
+          python -m twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/*
diff --git a/pytest.ini b/pytest.ini
@@ -1,3 +1,4 @@
 [pytest]
 norecursedirs = '.*', 'dist', 'CVS', '_darcs', '{arch}', '*.egg', 'venv', 'assets'
 testpaths = 'test'
+markers = docker_runner: Mark tests that require Docker to run.
diff --git a/test/functional/command/dockerfiles/Dockerfile b/test/functional/command/dockerfiles/Dockerfile
@@ -8,4 +8,4 @@ RUN apt-get update \
         python3-venv \
     && rm -rf /var/lib/apt/lists/*
 COPY . /root/conan-io
-RUN cd /root/conan-io && pip install -e .
+RUN cd /root/conan-io && pip install -e .