From e9eb9979041a958d1eec6047873b54bacf4a4734 Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 1 Oct 2024 15:49:06 +0200 Subject: [PATCH 01/39] add github actions --- .ci/__init__.py | 0 .ci/bump_dev_version.py | 22 --- .ci/docker/conan-tests | 152 ++++++++++++++++++++ .ci/jenkins/testsv2.jenkins | 184 ------------------------- .github/workflows/linux-tests.yml | 148 ++++++++++++++++++++ pytest.ini | 1 + test/functional/command/runner_test.py | 10 ++ 7 files changed, 311 insertions(+), 206 deletions(-) delete mode 100644 .ci/__init__.py delete mode 100644 .ci/bump_dev_version.py create mode 100644 .ci/docker/conan-tests delete mode 100644 .ci/jenkins/testsv2.jenkins create mode 100644 .github/workflows/linux-tests.yml diff --git a/.ci/__init__.py b/.ci/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/.ci/bump_dev_version.py b/.ci/bump_dev_version.py deleted file mode 100644 index e3f802833ea..00000000000 --- a/.ci/bump_dev_version.py +++ /dev/null @@ -1,22 +0,0 @@ -import os -import time - - -def replace_in_file(file_path, search, replace): - with open(file_path, "r") as handle: - content = handle.read() - if search not in content: - raise Exception("Incorrect development version in conans/__init__.py") - content = content.replace(search, replace) - content = content.encode("utf-8") - with open(file_path, "wb") as handle: - handle.write(content) - -def bump_dev(): - vfile = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../conans/__init__.py") - snapshot = "%s" % int(time.time()) - replace_in_file(vfile, "-dev'", "-dev%s'" % snapshot) - - -if __name__ == "__main__": - bump_dev() diff --git a/.ci/docker/conan-tests b/.ci/docker/conan-tests new file mode 100644 index 00000000000..218be27efdd --- /dev/null +++ b/.ci/docker/conan-tests @@ -0,0 +1,152 @@ +FROM ubuntu:24.04 + +LABEL maintainer="Conan.io " + +ARG DOCKER_GID=999 + +ENV DEBIAN_FRONTEND=noninteractive + +ENV PY36=3.6.15 \ + PY38=3.8.6 \ + PY39=3.9.2 \ + PY312=3.12.3 \ + CMAKE_3_15_7=/usr/share/cmake-3.15.7/bin/cmake \ + CMAKE_3_16_9=/usr/share/cmake-3.16.9/bin/cmake \ + CMAKE_3_17_5=/usr/share/cmake-3.17.5/bin/cmake \ + CMAKE_3_19_7=/usr/share/cmake-3.19.7/bin/cmake \ + CMAKE_3_23_5=/usr/share/cmake-3.23.5/bin/cmake \ + GCC_9=/usr/bin/gcc-9 \ + GXX_9=/usr/bin/g++-9 \ + GCC_11=/usr/bin/gcc-11 \ + GXX_11=/usr/bin/g++-11 \ + CLANG_14=/usr/bin/clang-14 \ + CLANGXX_14=/usr/bin/clang++-14 \ + BAZEL_6_3_2=6.3.2 \ + BAZEL_7_1_2=7.1.2 + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + software-properties-common \ + build-essential \ + libtool \ + automake \ + autoconf \ + pkg-config \ + gettext \ + git \ + curl \ + make \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + wget \ + llvm \ + libncurses5-dev \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + libffi-dev \ + liblzma-dev \ + python3-openssl \ + ca-certificates \ + sudo \ + tar \ + linux-libc-dev \ + subversion \ + subversion-tools \ + ninja-build \ + gcc-9 \ + g++-9 \ + gcc-11 \ + g++-11 \ + clang-14 \ + clang++-14 \ + fish \ + gcc-arm-linux-gnueabihf \ + g++-arm-linux-gnueabihf \ + unzip \ + apt-transport-https \ + gnupg-agent \ + gcc-9-multilib \ + g++-9-multilib \ + gcc-11-multilib \ + g++-11-multilib \ + scons && \ + # fix: asm/errno.h: No such file or directory + ln -s /usr/include/asm-generic/ /usr/include/asm && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + add-apt-repository -y ppa:fish-shell/release-3 && \ + apt-get update && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu jammy stable" && \ + apt-get update && \ + apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io && \ + rm -rf /var/lib/apt/lists/* + +RUN useradd -m -s /bin/bash conan && \ + echo 'conan ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +ENV HOME /home/conan +ENV PYENV_ROOT $HOME/.pyenv +ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:/usr/bin:/bin:$PATH + +RUN curl https://pyenv.run | bash && \ + pyenv install $PY36 && \ + pyenv install $PY38 && \ + pyenv install $PY39 && \ + pyenv install $PY312 && \ + pyenv global $PY39 && \ + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python get-pip.py && \ + rm get-pip.py + +RUN chown -R conan:conan $HOME + +USER root + +RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.7/cmake-3.15.7-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.15.7-Linux-x86_64.tar.gz && mv cmake-3.15.7-Linux-x86_64 /usr/share/cmake-3.15.7 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.16.9/cmake-3.16.9-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.16.9-Linux-x86_64.tar.gz && mv cmake-3.16.9-Linux-x86_64 /usr/share/cmake-3.16.9 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.17.5-Linux-x86_64.tar.gz && mv cmake-3.17.5-Linux-x86_64 /usr/share/cmake-3.17.5 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.19.7/cmake-3.19.7-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.19.7-Linux-x86_64.tar.gz && mv cmake-3.19.7-Linux-x86_64 /usr/share/cmake-3.19.7 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.23.5/cmake-3.23.5-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.23.5-Linux-x86_64.tar.gz && mv cmake-3.23.5-linux-x86_64/ /usr/share/cmake-3.23.5 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_15_7 10 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_16_9 20 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_17_5 30 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_19_7 40 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_23_5 50 + +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 10 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 30 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 10 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 30 && \ + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 10 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 10 && \ + update-alternatives --set gcc /usr/bin/gcc-9 && \ + update-alternatives --set g++ /usr/bin/g++-9 && \ + update-alternatives --set clang /usr/bin/clang-14 && \ + update-alternatives --set clang++ /usr/bin/clang++-14 + +RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \ + wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_6_3_2}/bazel-${BAZEL_6_3_2}-linux-x86_64 && \ + chmod +x bazel-${BAZEL_6_3_2}-linux-x86_64 && \ + mv bazel-${BAZEL_6_3_2}-linux-x86_64 /usr/share/bazel-$BAZEL_6_3_2/bin/bazel && \ + mkdir -p /usr/share/bazel-$BAZEL_7_1_2/bin && \ + wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_7_1_2}/bazel-${BAZEL_7_1_2}-linux-x86_64 && \ + chmod +x bazel-${BAZEL_7_1_2}-linux-x86_64 && \ + mv bazel-${BAZEL_7_1_2}-linux-x86_64 /usr/share/bazel-$BAZEL_7_1_2/bin/bazel + +RUN groupadd -g $DOCKER_GID docker || groupmod -g $DOCKER_GID docker + +RUN usermod -aG docker conan + +USER conan +WORKDIR $HOME + +CMD ["/bin/bash"] diff --git a/.ci/jenkins/testsv2.jenkins b/.ci/jenkins/testsv2.jenkins deleted file mode 100644 index 01bc871790c..00000000000 --- a/.ci/jenkins/testsv2.jenkins +++ /dev/null @@ -1,184 +0,0 @@ -def branchName = env.BRANCH_NAME -def jobName = env.JOB_NAME - -void cancelPrevious() { - stage("Cancelling previous") { - def buildNumber = env.BUILD_NUMBER as int - if (buildNumber > 1) milestone(buildNumber - 1) - milestone(buildNumber) - } -} - -String getTempFolder(String nodeName) { - if (nodeName == "Windows") { return "C:/J/t_v2/" } - return "/tmp/" -} - -List getConfigurations(String moduleName, String branchName, String jobName) { - def configs = [] - - String LinuxNode - if (moduleName == "test/unittests" || moduleName == "test/integration") { - LinuxNode = "LinuxUnittests" - } else if (moduleName == "test/functional") { - LinuxNode = "LinuxFunctional" - } - - configs.add([node: LinuxNode, pyvers: ["PY312"]]) - - if (branchName =~ /(^PR-.*)/) { - configs.add([node: LinuxNode, pyvers: ["PY38"]]) - configs.add([node: "Windows", pyvers: ["PY38"]]) - configs.add([node: "M2Macos", pyvers: ["PY38"]]) - } - else if (jobName == "ConanNightlyv2" || branchName =~ /^release.*/) { - configs.add([node: LinuxNode, pyvers: ["PY36", "PY38"]]) - configs.add([node: "Windows", pyvers: ["PY36", "PY38"]]) - configs.add([node: "M2Macos", pyvers: ["PY36", "PY38"]]) - } - else if (branchName == "develop2") { - configs.add([node: LinuxNode, pyvers: ["PY36", "PY38"]]) - configs.add([node: "Windows", pyvers: ["PY36", "PY38"]]) - configs.add([node: "M2Macos", pyvers: ["PY36", "PY38"]]) - } - return configs -} - -boolean shouldTestServer(String moduleName) { - // run conan_server --help after running functional tests - if (moduleName == "test/functional") { - return true - } - return false -} - -private Closure runTests(String nodeName, String pythonVersion, String module) { - def ret = { - node(nodeName) { - stage("${nodeName} - ${pythonVersion}") { - def scmVars = checkout scm - def commit = scmVars["GIT_COMMIT"].substring(0, 4) - def workDir = getTempFolder(nodeName) + "${commit}/${pythonVersion}/" - def venvName = "env_${env.BUILD_NUMBER}_${pythonVersion}_${commit}" - def pipArguments = "--no-cache-dir --timeout 30" - def pipInstalls = "python -m pip install -r conans/requirements.txt ${pipArguments} && " + - "python -m pip install -r conans/requirements_server.txt ${pipArguments} && " + - "python -m pip install -r conans/requirements_dev.txt ${pipArguments} && " + - "python -m pip install meson ${pipArguments} && " - def conanToolInstall = "python -m pip install . && conan --version && conan --help && python --version && " - def launchTests = "python -m pytest ${module} -n=4 --durations=20 -rs" - if (shouldTestServer(module)) { - def rm = (nodeName == "Windows") ? "del" : "rm" - def mv = (nodeName == "Windows") ? "move" : "mv" - def conanServerToolInstall = "${rm} setup.py && ${mv} setup_server.py setup.py && python -m pip install . && conan_server --help" - launchTests = launchTests + ' && ' + conanServerToolInstall - } - if (nodeName.indexOf("Linux") >= 0) { - try { - // we only test scons in Linux - pipInstalls = pipInstalls + "python -m pip install scons ${pipArguments} && " - def sourcedir = "/home/conan/" - def pyenvdir = "${sourcedir}.pyenv" - def launchCommand = "export PYENV_ROOT=${pyenvdir} && " + - "export PATH=\"${pyenvdir}/versions/\$${pythonVersion}/bin:${pyenvdir}/bin:\$PATH\" && " + - "pyenv global \$${pythonVersion} && " + - pipInstalls + - conanToolInstall + - launchTests - sh(script: launchCommand) - } finally {} - } else if (nodeName == "M2Macos") { - def localDir = "/Users/jenkins" - withEnv(["CONAN_TEST_FOLDER=${workDir}", "PY27=2.7.18", "PY36=3.6.15", "PY38=3.8.12", "PY39=3.9.10", - "PYENV_ROOT=${localDir}/.pyenv", "PATH+EXTRA=${localDir}/.pyenv/shims:/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"]) { - try { - sh(script: "mkdir -p ${workDir}") - def pythonLocation = "${localDir}/.pyenv/versions/\$${pythonVersion}/bin/python" - def configVenv = "${pythonLocation} -m virtualenv --python ${pythonLocation} ${workDir}${venvName} && " + - "source ${workDir}${venvName}/bin/activate && python --version && " - def launchCommand = configVenv + pipInstalls + conanToolInstall + launchTests - sh(script: launchCommand) - } finally { - sh(script: "rm -rf ${workDir}") - } - } - } else if (nodeName == "Windows") { - withEnv(["CONAN_TEST_FOLDER=${workDir}", "CONAN_BASH_PATH=c:/tools/msys64/usr/bin/bash", - "PY27=Python27", "PY36=Python36", "PY38=Python38-64"]) { - try { - bat(script: "if not exist \"${workDir}\" mkdir \"${workDir}\"") - def pythonLocation = "C:/%${pythonVersion}%/python.exe" - def configVenv = "virtualenv --python ${pythonLocation} ${workDir}${venvName} && " + - "${workDir}${venvName}/Scripts/activate && python --version && " - def launchCommand = configVenv + pipInstalls + conanToolInstall + launchTests - bat(script: launchCommand) - } finally { - bat(script: "rd /s /q \"${workDir}\"") - } - } - } - cleanWs() - } - } - } - return ret -} - -void runTestsModule(String moduleName, String branchName, String jobName) { - def configs = getConfigurations(moduleName, branchName, jobName) - def paralellRuns = [:] - configs.each { config -> - def testKind = moduleName.split("/").last() - config["pyvers"].each { pyver -> - paralellRuns["${testKind} - ${config['node']} - ${pyver}"] = runTests(config["node"], pyver, moduleName) - } - } - parallel(paralellRuns) -} - -def testModules = ["test/unittests", "test/integration", "test/functional"] - -void deployToPypiTest() { - node("LinuxFunctional") { - try { - def scmVars = checkout scm - checkout scm - stage("Deploy conan to pypitesting") { - withCredentials([usernamePassword(credentialsId: 'PYPITEST_CONAN_CREDENTIALS', usernameVariable: 'TWINE_USERNAME', passwordVariable: 'TWINE_PASSWORD')]) { - def deployConan = "python -m pip install twine && " + - "python .ci/bump_dev_version.py && " + - "rm -rf dist/ && python setup.py sdist && " + - "python -m twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/*" - sh(script: deployConan) - } - } - stage("Deploy conan-server to pypitesting") { - withCredentials([usernamePassword(credentialsId: 'PYPITEST_CONAN_SERVER_CREDENTIALS', usernameVariable: 'TWINE_USERNAME', passwordVariable: 'TWINE_PASSWORD')]) { - def deployServer = "rm -rf dist/ rm setup.py && mv setup_server.py setup.py && python setup.py sdist && " + - "python -m twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/*" - sh(script: deployServer) - } - } - } finally {} - } -} - -try { - cancelPrevious() - - testModules.each { moduleName -> - runTestsModule(moduleName, branchName, jobName) - } - - if (branchName == "develop2") { - deployToPypiTest() - } -} -catch(e){ - if (branchName == "develop2") { - def subject = "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'" - def summary = "${subject} (${env.BUILD_URL}), Conan Branch: ${branchName}" - slackSend (color: '#FF0000', message: summary) - } - throw e -} diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml new file mode 100644 index 00000000000..7465cfaa990 --- /dev/null +++ b/.github/workflows/linux-tests.yml @@ -0,0 +1,148 @@ +name: Linux tests + +on: + push: + branches: + - develop2 + - release/* + pull_request: + branches: + - '*' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build_container: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Calculate Dockerfile checksum + id: dockerfile_hash + run: | + DOCKERFILE_HASH=$(find ./.ci/docker/conan-tests -type f -exec sha256sum {} \; | sha256sum | cut -d' ' -f1) + echo "hash=$DOCKERFILE_HASH" >> $GITHUB_OUTPUT + + - name: Cache Docker image + id: cache_docker_image + uses: actions/cache@v3 + with: + path: docker_image_cache + key: ${{ runner.os }}-docker-${{ steps.dockerfile_hash.outputs.hash }} + + - name: Load Docker image from cache + if: steps.cache_docker_image.outputs.cache-hit == 'true' + run: docker load -i docker_image_cache/conan-tests.tar + + - name: Get Docker group GID + if: steps.cache_docker_image.outputs.cache-hit != 'true' + id: docker_gid + run: | + DOCKER_GID=$(getent group docker | cut -d: -f3) + echo "docker_gid=$DOCKER_GID" >> $GITHUB_OUTPUT + + - name: Build Docker image (if not cached) + if: steps.cache_docker_image.outputs.cache-hit != 'true' + run: | + docker build \ + --build-arg DOCKER_GID=${{ steps.docker_gid.outputs.docker_gid }} \ + -t ghcr.io/${{ github.repository_owner }}/conan-tests:latest \ + -f ./.ci/docker/conan-tests . + mkdir -p docker_image_cache + docker save ghcr.io/${{ github.repository_owner }}/conan-tests:latest -o docker_image_cache/conan-tests.tar + + - name: Push Docker image to GHCR + if: steps.cache_docker_image.outputs.cache-hit != 'true' + run: docker push ghcr.io/${{ github.repository_owner }}/conan-tests:latest + + # linux: + # needs: build_container + # runs-on: ubuntu-latest + # container: + # image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest + # options: --user conan + # strategy: + # matrix: + # python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] + # test-type: ['unittests', 'integration', 'functional'] + # name: Python ${{ matrix.python-version }} - ${{ matrix.test-type }} + # steps: + # - name: Checkout code + # uses: actions/checkout@v3 + + # - name: Set up Python ${{ matrix.python-version }} + # run: | + # pyenv global ${{ matrix.python-version }} + # python --version + + # - name: Cache pip + # uses: actions/cache@v3 + # with: + # path: ~/.cache/pip + # key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }} + + # - name: Install dependencies + # run: | + # pip install --upgrade pip + # pip install -r conans/requirements.txt + # pip install -r conans/requirements_dev.txt + # pip install -r conans/requirements_server.txt + # pip install meson + + # - name: Run tests + # shell: bash + # run: | + # if [ "${{ matrix.test-type }}" == "unittests" ]; then + # pytest test/unittests --durations=20 -n 4 -rs + # elif [ "${{ matrix.test-type }}" == "integration" ]; then + # pytest test/integration --durations=20 -n 4 -rs + # elif [ "${{ matrix.test-type }}" == "functional" ]; then + # pytest test/functional --durations=20 -n 4 -rs + # fi + + linux_docker_tests: + needs: build_container + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest + options: --user conan -v /var/run/docker.sock:/var/run/docker.sock + strategy: + matrix: + python-version: ['3.12.3'] + name: Docker Runner Tests - Python ${{ matrix.python-version }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull Docker image + run: docker pull ghcr.io/${{ github.repository_owner }}/conan-tests:latest + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -r conans/requirements.txt + pip install -r conans/requirements_dev.txt + pip install -r conans/requirements_server.txt + pip install meson + + - name: Run tests + shell: bash + run: | + pytest -m docker_runner --durations=20 -rs diff --git a/pytest.ini b/pytest.ini index a69be4e9060..26456fc871e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,4 @@ [pytest] norecursedirs = '.*', 'dist', 'CVS', '_darcs', '{arch}', '*.egg', 'venv', 'assets' testpaths = 'test' +markers = docker_runner: Mark tests that require Docker to run. diff --git a/test/functional/command/runner_test.py b/test/functional/command/runner_test.py index fb601288aeb..4d4f58fa610 100644 --- a/test/functional/command/runner_test.py +++ b/test/functional/command/runner_test.py @@ -36,6 +36,7 @@ def dockerfile_path(name=None): return path +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_cache_shared(): """ @@ -79,6 +80,7 @@ def test_create_docker_runner_cache_shared(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_cache_shared_profile_from_cache(): """ @@ -122,6 +124,7 @@ def test_create_docker_runner_cache_shared_profile_from_cache(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_cache_shared_profile_folder(): """ @@ -165,6 +168,7 @@ def test_create_docker_runner_cache_shared_profile_folder(): assert "[100%] Built target example" in client.out assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_dockerfile_folder_path(): """ @@ -235,6 +239,7 @@ def test_create_docker_runner_dockerfile_folder_path(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_profile_default_folder(): """ @@ -280,6 +285,7 @@ def test_create_docker_runner_profile_default_folder(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_dockerfile_file_path(): """ @@ -324,6 +330,7 @@ def test_create_docker_runner_dockerfile_file_path(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") @pytest.mark.parametrize("build_type,shared", [("Release", False), ("Debug", True)]) @pytest.mark.tool("ninja") @@ -389,6 +396,7 @@ def package(self): assert 'cmake -G "Ninja"' in client.out assert "main: {}!".format(build_type) in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_from_configfile(): """ @@ -443,6 +451,7 @@ def test_create_docker_runner_from_configfile(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_from_configfile_with_args(): """ @@ -499,6 +508,7 @@ def test_create_docker_runner_from_configfile_with_args(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_default_build_profile(): """ From 8878e44440ca3d3b76ad6e8d5f361581b596ab1c Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 07:52:46 +0200 Subject: [PATCH 02/39] output info --- conan/internal/runner/docker.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index b9f9ab8f1e4..b48c7be2c3a 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -146,6 +146,17 @@ def run(self): security_opt=self.configfile.run.security_opt, detach=True, auto_remove=False) + print(self.image, + "/bin/bash -c 'while true; do sleep 30; done;'", + f"name={self.name}", + f"volumes={volumes}", + f"environment={environment}", + f"user={self.configfile.run.user}", + f"privileged={self.configfile.run.privileged}", + f"cap_add={self.configfile.run.cap_add}", + f"security_opt={self.configfile.run.security_opt}", + f"detach={True}", + f"auto_remove={False}") _docker_info(f'Container {self.name} running') except Exception as e: raise ConanException(f'Imposible to run the container "{self.name}" with image "{self.image}"' From 4da2d7b3e9f666acd044d54f10acc0fdd79c7d60 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 08:59:11 +0200 Subject: [PATCH 03/39] add user --- test/functional/command/runner_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/functional/command/runner_test.py b/test/functional/command/runner_test.py index 4d4f58fa610..0c5748eb0e8 100644 --- a/test/functional/command/runner_test.py +++ b/test/functional/command/runner_test.py @@ -410,6 +410,7 @@ def test_create_docker_runner_from_configfile(): build_context: {conan_base_path()} run: name: my-custom-conan-runner-container + containerUser: conan """) client.save({"configfile.yaml": configfile}) @@ -467,6 +468,7 @@ def test_create_docker_runner_from_configfile_with_args(): BASE_IMAGE: ubuntu:22.04 run: name: my-conan-runner-container-with-args + containerUser: conan """) client.save({"configfile.yaml": configfile}) From 7035493d970a00463f968eaa5a1eee447e6aa968 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 09:10:03 +0200 Subject: [PATCH 04/39] revert user --- test/functional/command/runner_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/functional/command/runner_test.py b/test/functional/command/runner_test.py index 0c5748eb0e8..4d4f58fa610 100644 --- a/test/functional/command/runner_test.py +++ b/test/functional/command/runner_test.py @@ -410,7 +410,6 @@ def test_create_docker_runner_from_configfile(): build_context: {conan_base_path()} run: name: my-custom-conan-runner-container - containerUser: conan """) client.save({"configfile.yaml": configfile}) @@ -468,7 +467,6 @@ def test_create_docker_runner_from_configfile_with_args(): BASE_IMAGE: ubuntu:22.04 run: name: my-conan-runner-container-with-args - containerUser: conan """) client.save({"configfile.yaml": configfile}) From 3f643c46d069449ed0bab2c87b095591db3a374f Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 10:08:50 +0200 Subject: [PATCH 05/39] use conan-tests for docker testing --- test/functional/command/dockerfiles/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/command/dockerfiles/Dockerfile b/test/functional/command/dockerfiles/Dockerfile index 068de9a2efd..0daf78f67c6 100644 --- a/test/functional/command/dockerfiles/Dockerfile +++ b/test/functional/command/dockerfiles/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 +FROM conan-tests:latest RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ From 4010cc7d1fa9c2a05afc8b20d3c8b2034e4f107a Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 10:23:38 +0200 Subject: [PATCH 06/39] get docker id --- .github/workflows/linux-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 7465cfaa990..13774641b05 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -45,11 +45,11 @@ jobs: run: docker load -i docker_image_cache/conan-tests.tar - name: Get Docker group GID - if: steps.cache_docker_image.outputs.cache-hit != 'true' + #if: steps.cache_docker_image.outputs.cache-hit != 'true' id: docker_gid run: | DOCKER_GID=$(getent group docker | cut -d: -f3) - echo "docker_gid=$DOCKER_GID" >> $GITHUB_OUTPUT + echo "docker_gid=$DOCKER_GID" - name: Build Docker image (if not cached) if: steps.cache_docker_image.outputs.cache-hit != 'true' From 1380617ac6dccda84b5e2abe823f618ee86059b1 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 10:23:53 +0200 Subject: [PATCH 07/39] revert --- .github/workflows/linux-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 13774641b05..7465cfaa990 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -45,11 +45,11 @@ jobs: run: docker load -i docker_image_cache/conan-tests.tar - name: Get Docker group GID - #if: steps.cache_docker_image.outputs.cache-hit != 'true' + if: steps.cache_docker_image.outputs.cache-hit != 'true' id: docker_gid run: | DOCKER_GID=$(getent group docker | cut -d: -f3) - echo "docker_gid=$DOCKER_GID" + echo "docker_gid=$DOCKER_GID" >> $GITHUB_OUTPUT - name: Build Docker image (if not cached) if: steps.cache_docker_image.outputs.cache-hit != 'true' From 3528b566772832911faa37ac6455464b8400dba2 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 10:36:24 +0200 Subject: [PATCH 08/39] try changes --- test/functional/command/dockerfiles/Dockerfile | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test/functional/command/dockerfiles/Dockerfile b/test/functional/command/dockerfiles/Dockerfile index 0daf78f67c6..bd28656d0f9 100644 --- a/test/functional/command/dockerfiles/Dockerfile +++ b/test/functional/command/dockerfiles/Dockerfile @@ -1,4 +1,5 @@ -FROM conan-tests:latest +FROM ubuntu:22.04 + RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ @@ -6,6 +7,14 @@ RUN apt-get update \ python3 \ python3-pip \ python3-venv \ + sudo \ && rm -rf /var/lib/apt/lists/* + +ARG DOCKER_GID=127 + +RUN groupadd -g $DOCKER_GID docker || groupmod -g $DOCKER_GID docker + +RUN usermod -aG docker root + COPY . /root/conan-io -RUN cd /root/conan-io && pip install -e . \ No newline at end of file +RUN cd /root/conan-io && pip install -e . From 32540cce45383decb218d2ff1a69f2e2cabba664 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 10:39:38 +0200 Subject: [PATCH 09/39] update cache and checkout --- .github/workflows/linux-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 7465cfaa990..6ec82747c70 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Log in to GitHub Container Registry uses: docker/login-action@v3 @@ -35,7 +35,7 @@ jobs: - name: Cache Docker image id: cache_docker_image - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: docker_image_cache key: ${{ runner.os }}-docker-${{ steps.dockerfile_hash.outputs.hash }} @@ -78,7 +78,7 @@ jobs: # name: Python ${{ matrix.python-version }} - ${{ matrix.test-type }} # steps: # - name: Checkout code - # uses: actions/checkout@v3 + # uses: actions/checkout@v4 # - name: Set up Python ${{ matrix.python-version }} # run: | @@ -86,7 +86,7 @@ jobs: # python --version # - name: Cache pip - # uses: actions/cache@v3 + # uses: actions/cache@v4 # with: # path: ~/.cache/pip # key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }} @@ -122,7 +122,7 @@ jobs: name: Docker Runner Tests - Python ${{ matrix.python-version }} steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Log in to GitHub Container Registry uses: docker/login-action@v3 From 1c76a7622a240c99f01fb4ea1411d85936a0affd Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 10:47:08 +0200 Subject: [PATCH 10/39] revert changes --- test/functional/command/dockerfiles/Dockerfile | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test/functional/command/dockerfiles/Dockerfile b/test/functional/command/dockerfiles/Dockerfile index bd28656d0f9..a90abd2b9cf 100644 --- a/test/functional/command/dockerfiles/Dockerfile +++ b/test/functional/command/dockerfiles/Dockerfile @@ -1,5 +1,4 @@ FROM ubuntu:22.04 - RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ @@ -7,14 +6,6 @@ RUN apt-get update \ python3 \ python3-pip \ python3-venv \ - sudo \ && rm -rf /var/lib/apt/lists/* - -ARG DOCKER_GID=127 - -RUN groupadd -g $DOCKER_GID docker || groupmod -g $DOCKER_GID docker - -RUN usermod -aG docker root - COPY . /root/conan-io RUN cd /root/conan-io && pip install -e . From e5307a725879fa94df4353496a3e3032cfbabbdf Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 12:05:30 +0200 Subject: [PATCH 11/39] do not use spaces --- test/functional/command/runner_test.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/functional/command/runner_test.py b/test/functional/command/runner_test.py index 4d4f58fa610..048d74d1b2d 100644 --- a/test/functional/command/runner_test.py +++ b/test/functional/command/runner_test.py @@ -42,7 +42,7 @@ def test_create_docker_runner_cache_shared(): """ Tests the ``conan create . `` """ - client = TestClient() + client = TestClient(path_with_spaces=False) profile_build = textwrap.dedent(f"""\ [settings] arch=x86_64 @@ -86,7 +86,7 @@ def test_create_docker_runner_cache_shared_profile_from_cache(): """ Tests the ``conan create . `` """ - client = TestClient() + client = TestClient(path_with_spaces=False) profile_build = textwrap.dedent(f"""\ [settings] arch=x86_64 @@ -130,7 +130,7 @@ def test_create_docker_runner_cache_shared_profile_folder(): """ Tests the ``conan create . `` """ - client = TestClient() + client = TestClient(path_with_spaces=False) profile_build = textwrap.dedent(f"""\ [settings] arch=x86_64 @@ -174,7 +174,7 @@ def test_create_docker_runner_dockerfile_folder_path(): """ Tests the ``conan create . `` """ - client = TestClient() + client = TestClient(path_with_spaces=False) profile_build = textwrap.dedent(f"""\ [settings] arch=x86_64 @@ -245,7 +245,7 @@ def test_create_docker_runner_profile_default_folder(): """ Tests the ``conan create . `` """ - client = TestClient() + client = TestClient(path_with_spaces=False) profile_build = textwrap.dedent(f"""\ [settings] arch=x86_64 @@ -291,7 +291,7 @@ def test_create_docker_runner_dockerfile_file_path(): """ Tests the ``conan create . `` """ - client = TestClient() + client = TestClient(path_with_spaces=False) profile_build = textwrap.dedent(f"""\ [settings] arch=x86_64 @@ -363,7 +363,7 @@ def package(self): cmake.install() """) - client = TestClient(path_with_spaces=False) + client = TestClient(path_with_spaces=Falsepath_with_spaces=False) client.save({'conanfile.py': conanfile, "CMakeLists.txt": gen_cmakelists(libsources=["hello.cpp"], appsources=["main.cpp"], @@ -402,7 +402,7 @@ def test_create_docker_runner_from_configfile(): """ Tests the ``conan create . `` """ - client = TestClient() + client = TestClient(path_with_spaces=False) configfile = textwrap.dedent(f""" image: conan-runner-default-test build: @@ -457,7 +457,7 @@ def test_create_docker_runner_from_configfile_with_args(): """ Tests the ``conan create . `` """ - client = TestClient() + client = TestClient(path_with_spaces=False) configfile = textwrap.dedent(f""" image: conan-runner-default-test-with-args build: @@ -514,7 +514,7 @@ def test_create_docker_runner_default_build_profile(): """ Tests the ``conan create . `` """ - client = TestClient() + client = TestClient(path_with_spaces=False) profile_host = textwrap.dedent(f"""\ [settings] From 8e4435a6e60bb6caf49349d95e2158330ecf8148 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 12:28:06 +0200 Subject: [PATCH 12/39] fix --- test/functional/command/runner_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/command/runner_test.py b/test/functional/command/runner_test.py index 048d74d1b2d..5d8a144713d 100644 --- a/test/functional/command/runner_test.py +++ b/test/functional/command/runner_test.py @@ -363,7 +363,7 @@ def package(self): cmake.install() """) - client = TestClient(path_with_spaces=Falsepath_with_spaces=False) + client = TestClient(path_with_spaces=False) client.save({'conanfile.py': conanfile, "CMakeLists.txt": gen_cmakelists(libsources=["hello.cpp"], appsources=["main.cpp"], From 27ccfea699622a84c8cb4b9f22ea89ba77d5d2f9 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 15:09:09 +0200 Subject: [PATCH 13/39] wip --- conan/internal/runner/docker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index b48c7be2c3a..90f46fbf2b2 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -113,6 +113,7 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar # Update conan command and some paths to run inside the container raw_args[raw_args.index(args.path)] = self.abs_docker_path self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json']) + self.command = "pwd && ls && ls /root && ls /root/.conan2 && " + self.command def run(self): """ From 02d00ae8de6683893e2e3ac73a9fe813ef9690a2 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 15:35:31 +0200 Subject: [PATCH 14/39] wip --- conan/internal/runner/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 90f46fbf2b2..72646cbb087 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -113,7 +113,7 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar # Update conan command and some paths to run inside the container raw_args[raw_args.index(args.path)] = self.abs_docker_path self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json']) - self.command = "pwd && ls && ls /root && ls /root/.conan2 && " + self.command + self.command = "pwd && ls /root/conanrunner/pathwithoutspaces && " + self.command def run(self): """ From 597bc0074946e78e53e696ab2f4988abe4041bf4 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 16:24:08 +0200 Subject: [PATCH 15/39] wip --- .ci/docker/conan-tests | 2 +- .github/workflows/linux-tests.yml | 2 +- conan/internal/runner/docker.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/docker/conan-tests b/.ci/docker/conan-tests index 218be27efdd..92ed3bba756 100644 --- a/.ci/docker/conan-tests +++ b/.ci/docker/conan-tests @@ -144,7 +144,7 @@ RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \ RUN groupadd -g $DOCKER_GID docker || groupmod -g $DOCKER_GID docker -RUN usermod -aG docker conan +RUN usermod -aG docker root USER conan WORKDIR $HOME diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 6ec82747c70..52dd8a3e44f 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -115,7 +115,7 @@ jobs: runs-on: ubuntu-latest container: image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest - options: --user conan -v /var/run/docker.sock:/var/run/docker.sock + options: -v /var/run/docker.sock:/var/run/docker.sock strategy: matrix: python-version: ['3.12.3'] diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 72646cbb087..8a78c63b542 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -113,7 +113,7 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar # Update conan command and some paths to run inside the container raw_args[raw_args.index(args.path)] = self.abs_docker_path self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json']) - self.command = "pwd && ls /root/conanrunner/pathwithoutspaces && " + self.command + self.command = "pwd && ls /root/conanrunner/pathwithoutspaces && ls /root/conan-io && " + self.command def run(self): """ From 58d175d9772fa0b86e2c3db5d08ae71d9af6f116 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 16:39:40 +0200 Subject: [PATCH 16/39] revert changes --- .ci/docker/conan-tests | 2 +- .github/workflows/linux-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/docker/conan-tests b/.ci/docker/conan-tests index 92ed3bba756..218be27efdd 100644 --- a/.ci/docker/conan-tests +++ b/.ci/docker/conan-tests @@ -144,7 +144,7 @@ RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \ RUN groupadd -g $DOCKER_GID docker || groupmod -g $DOCKER_GID docker -RUN usermod -aG docker root +RUN usermod -aG docker conan USER conan WORKDIR $HOME diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 52dd8a3e44f..6ec82747c70 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -115,7 +115,7 @@ jobs: runs-on: ubuntu-latest container: image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest - options: -v /var/run/docker.sock:/var/run/docker.sock + options: --user conan -v /var/run/docker.sock:/var/run/docker.sock strategy: matrix: python-version: ['3.12.3'] From dc3bd8a8b1552134f13c4cf373edb53dd32db260 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 17:49:45 +0200 Subject: [PATCH 17/39] Revert "revert changes" This reverts commit 58d175d9772fa0b86e2c3db5d08ae71d9af6f116. --- .ci/docker/conan-tests | 2 +- .github/workflows/linux-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/docker/conan-tests b/.ci/docker/conan-tests index 218be27efdd..92ed3bba756 100644 --- a/.ci/docker/conan-tests +++ b/.ci/docker/conan-tests @@ -144,7 +144,7 @@ RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \ RUN groupadd -g $DOCKER_GID docker || groupmod -g $DOCKER_GID docker -RUN usermod -aG docker conan +RUN usermod -aG docker root USER conan WORKDIR $HOME diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 6ec82747c70..52dd8a3e44f 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -115,7 +115,7 @@ jobs: runs-on: ubuntu-latest container: image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest - options: --user conan -v /var/run/docker.sock:/var/run/docker.sock + options: -v /var/run/docker.sock:/var/run/docker.sock strategy: matrix: python-version: ['3.12.3'] From 8160d8f8092c7a8578185c71d3fd0dfc9750bfe4 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 2 Oct 2024 17:51:11 +0200 Subject: [PATCH 18/39] Revert "wip" This reverts commit 597bc0074946e78e53e696ab2f4988abe4041bf4. --- .ci/docker/conan-tests | 2 +- .github/workflows/linux-tests.yml | 2 +- conan/internal/runner/docker.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/docker/conan-tests b/.ci/docker/conan-tests index 92ed3bba756..218be27efdd 100644 --- a/.ci/docker/conan-tests +++ b/.ci/docker/conan-tests @@ -144,7 +144,7 @@ RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \ RUN groupadd -g $DOCKER_GID docker || groupmod -g $DOCKER_GID docker -RUN usermod -aG docker root +RUN usermod -aG docker conan USER conan WORKDIR $HOME diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 52dd8a3e44f..6ec82747c70 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -115,7 +115,7 @@ jobs: runs-on: ubuntu-latest container: image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest - options: -v /var/run/docker.sock:/var/run/docker.sock + options: --user conan -v /var/run/docker.sock:/var/run/docker.sock strategy: matrix: python-version: ['3.12.3'] diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 8a78c63b542..72646cbb087 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -113,7 +113,7 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar # Update conan command and some paths to run inside the container raw_args[raw_args.index(args.path)] = self.abs_docker_path self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json']) - self.command = "pwd && ls /root/conanrunner/pathwithoutspaces && ls /root/conan-io && " + self.command + self.command = "pwd && ls /root/conanrunner/pathwithoutspaces && " + self.command def run(self): """ From b85d82706f9ad9ce29038a57a5f3afd2abf46d63 Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 7 Oct 2024 16:20:09 +0200 Subject: [PATCH 19/39] try without image --- .github/workflows/linux-tests.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 6ec82747c70..684003dd75b 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -113,9 +113,6 @@ jobs: linux_docker_tests: needs: build_container runs-on: ubuntu-latest - container: - image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest - options: --user conan -v /var/run/docker.sock:/var/run/docker.sock strategy: matrix: python-version: ['3.12.3'] @@ -123,17 +120,6 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Pull Docker image - run: docker pull ghcr.io/${{ github.repository_owner }}/conan-tests:latest - - name: Install dependencies run: | pip install --upgrade pip @@ -141,7 +127,6 @@ jobs: pip install -r conans/requirements_dev.txt pip install -r conans/requirements_server.txt pip install meson - - name: Run tests shell: bash run: | From d4c0ef5c0e82b47d04f438d37f18b093debc23fb Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 7 Oct 2024 16:27:17 +0200 Subject: [PATCH 20/39] remove ninja mark --- test/functional/command/runner_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/functional/command/runner_test.py b/test/functional/command/runner_test.py index 5d8a144713d..7dba8cf6874 100644 --- a/test/functional/command/runner_test.py +++ b/test/functional/command/runner_test.py @@ -333,7 +333,6 @@ def test_create_docker_runner_dockerfile_file_path(): @pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") @pytest.mark.parametrize("build_type,shared", [("Release", False), ("Debug", True)]) -@pytest.mark.tool("ninja") def test_create_docker_runner_with_ninja(build_type, shared): conanfile = textwrap.dedent(""" import os From 2fd4b1c223e67e50ca1417461f03a5dfe68351f0 Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 7 Oct 2024 16:40:30 +0200 Subject: [PATCH 21/39] wip --- .github/workflows/linux-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 684003dd75b..3f36f1dfdc8 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -126,7 +126,7 @@ jobs: pip install -r conans/requirements.txt pip install -r conans/requirements_dev.txt pip install -r conans/requirements_server.txt - pip install meson + pip install -r conans/requirements_runner.txt - name: Run tests shell: bash run: | From 0a7fda5b736c02920ae8830e81d798b05251e69f Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 7 Oct 2024 16:53:59 +0200 Subject: [PATCH 22/39] remove unused --- .ci/docker/conan-tests | 6 -- .github/workflows/linux-tests.yml | 90 ++++++++++++-------------- conan/internal/runner/docker.py | 12 ---- test/functional/command/runner_test.py | 20 +++--- 4 files changed, 51 insertions(+), 77 deletions(-) diff --git a/.ci/docker/conan-tests b/.ci/docker/conan-tests index 218be27efdd..d801bd19eff 100644 --- a/.ci/docker/conan-tests +++ b/.ci/docker/conan-tests @@ -2,8 +2,6 @@ FROM ubuntu:24.04 LABEL maintainer="Conan.io " -ARG DOCKER_GID=999 - ENV DEBIAN_FRONTEND=noninteractive ENV PY36=3.6.15 \ @@ -142,10 +140,6 @@ RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \ chmod +x bazel-${BAZEL_7_1_2}-linux-x86_64 && \ mv bazel-${BAZEL_7_1_2}-linux-x86_64 /usr/share/bazel-$BAZEL_7_1_2/bin/bazel -RUN groupadd -g $DOCKER_GID docker || groupmod -g $DOCKER_GID docker - -RUN usermod -aG docker conan - USER conan WORKDIR $HOME diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 3f36f1dfdc8..4be553120e8 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -44,18 +44,10 @@ jobs: if: steps.cache_docker_image.outputs.cache-hit == 'true' run: docker load -i docker_image_cache/conan-tests.tar - - name: Get Docker group GID - if: steps.cache_docker_image.outputs.cache-hit != 'true' - id: docker_gid - run: | - DOCKER_GID=$(getent group docker | cut -d: -f3) - echo "docker_gid=$DOCKER_GID" >> $GITHUB_OUTPUT - - name: Build Docker image (if not cached) if: steps.cache_docker_image.outputs.cache-hit != 'true' run: | docker build \ - --build-arg DOCKER_GID=${{ steps.docker_gid.outputs.docker_gid }} \ -t ghcr.io/${{ github.repository_owner }}/conan-tests:latest \ -f ./.ci/docker/conan-tests . mkdir -p docker_image_cache @@ -65,57 +57,57 @@ jobs: if: steps.cache_docker_image.outputs.cache-hit != 'true' run: docker push ghcr.io/${{ github.repository_owner }}/conan-tests:latest - # linux: - # needs: build_container - # runs-on: ubuntu-latest - # container: - # image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest - # options: --user conan - # strategy: - # matrix: - # python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] - # test-type: ['unittests', 'integration', 'functional'] - # name: Python ${{ matrix.python-version }} - ${{ matrix.test-type }} - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 + linux: + needs: build_container + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest + options: --user conan + strategy: + matrix: + python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] + test-type: ['unittests', 'integration', 'functional'] + name: Python ${{ matrix.python-version }} - ${{ matrix.test-type }} + steps: + - name: Checkout code + uses: actions/checkout@v4 - # - name: Set up Python ${{ matrix.python-version }} - # run: | - # pyenv global ${{ matrix.python-version }} - # python --version + - name: Set up Python ${{ matrix.python-version }} + run: | + pyenv global ${{ matrix.python-version }} + python --version - # - name: Cache pip - # uses: actions/cache@v4 - # with: - # path: ~/.cache/pip - # key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }} + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }} - # - name: Install dependencies - # run: | - # pip install --upgrade pip - # pip install -r conans/requirements.txt - # pip install -r conans/requirements_dev.txt - # pip install -r conans/requirements_server.txt - # pip install meson + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -r conans/requirements.txt + pip install -r conans/requirements_dev.txt + pip install -r conans/requirements_server.txt + pip install meson - # - name: Run tests - # shell: bash - # run: | - # if [ "${{ matrix.test-type }}" == "unittests" ]; then - # pytest test/unittests --durations=20 -n 4 -rs - # elif [ "${{ matrix.test-type }}" == "integration" ]; then - # pytest test/integration --durations=20 -n 4 -rs - # elif [ "${{ matrix.test-type }}" == "functional" ]; then - # pytest test/functional --durations=20 -n 4 -rs - # fi + - name: Run tests + shell: bash + run: | + if [ "${{ matrix.test-type }}" == "unittests" ]; then + pytest test/unittests --durations=20 -n 4 -rs + elif [ "${{ matrix.test-type }}" == "integration" ]; then + pytest test/integration --durations=20 -n 4 -rs + elif [ "${{ matrix.test-type }}" == "functional" ]; then + pytest test/functional --durations=20 -n 4 -rs + fi linux_docker_tests: needs: build_container runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.12.3'] + python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] name: Docker Runner Tests - Python ${{ matrix.python-version }} steps: - name: Checkout code diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 72646cbb087..b9f9ab8f1e4 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -113,7 +113,6 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar # Update conan command and some paths to run inside the container raw_args[raw_args.index(args.path)] = self.abs_docker_path self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json']) - self.command = "pwd && ls /root/conanrunner/pathwithoutspaces && " + self.command def run(self): """ @@ -147,17 +146,6 @@ def run(self): security_opt=self.configfile.run.security_opt, detach=True, auto_remove=False) - print(self.image, - "/bin/bash -c 'while true; do sleep 30; done;'", - f"name={self.name}", - f"volumes={volumes}", - f"environment={environment}", - f"user={self.configfile.run.user}", - f"privileged={self.configfile.run.privileged}", - f"cap_add={self.configfile.run.cap_add}", - f"security_opt={self.configfile.run.security_opt}", - f"detach={True}", - f"auto_remove={False}") _docker_info(f'Container {self.name} running') except Exception as e: raise ConanException(f'Imposible to run the container "{self.name}" with image "{self.image}"' diff --git a/test/functional/command/runner_test.py b/test/functional/command/runner_test.py index 7dba8cf6874..41c7387af74 100644 --- a/test/functional/command/runner_test.py +++ b/test/functional/command/runner_test.py @@ -42,7 +42,7 @@ def test_create_docker_runner_cache_shared(): """ Tests the ``conan create . `` """ - client = TestClient(path_with_spaces=False) + client = TestClient() profile_build = textwrap.dedent(f"""\ [settings] arch=x86_64 @@ -86,7 +86,7 @@ def test_create_docker_runner_cache_shared_profile_from_cache(): """ Tests the ``conan create . `` """ - client = TestClient(path_with_spaces=False) + client = TestClient() profile_build = textwrap.dedent(f"""\ [settings] arch=x86_64 @@ -130,7 +130,7 @@ def test_create_docker_runner_cache_shared_profile_folder(): """ Tests the ``conan create . `` """ - client = TestClient(path_with_spaces=False) + client = TestClient() profile_build = textwrap.dedent(f"""\ [settings] arch=x86_64 @@ -174,7 +174,7 @@ def test_create_docker_runner_dockerfile_folder_path(): """ Tests the ``conan create . `` """ - client = TestClient(path_with_spaces=False) + client = TestClient() profile_build = textwrap.dedent(f"""\ [settings] arch=x86_64 @@ -245,7 +245,7 @@ def test_create_docker_runner_profile_default_folder(): """ Tests the ``conan create . `` """ - client = TestClient(path_with_spaces=False) + client = TestClient() profile_build = textwrap.dedent(f"""\ [settings] arch=x86_64 @@ -291,7 +291,7 @@ def test_create_docker_runner_dockerfile_file_path(): """ Tests the ``conan create . `` """ - client = TestClient(path_with_spaces=False) + client = TestClient() profile_build = textwrap.dedent(f"""\ [settings] arch=x86_64 @@ -362,7 +362,7 @@ def package(self): cmake.install() """) - client = TestClient(path_with_spaces=False) + client = TestClient() client.save({'conanfile.py': conanfile, "CMakeLists.txt": gen_cmakelists(libsources=["hello.cpp"], appsources=["main.cpp"], @@ -401,7 +401,7 @@ def test_create_docker_runner_from_configfile(): """ Tests the ``conan create . `` """ - client = TestClient(path_with_spaces=False) + client = TestClient() configfile = textwrap.dedent(f""" image: conan-runner-default-test build: @@ -456,7 +456,7 @@ def test_create_docker_runner_from_configfile_with_args(): """ Tests the ``conan create . `` """ - client = TestClient(path_with_spaces=False) + client = TestClient() configfile = textwrap.dedent(f""" image: conan-runner-default-test-with-args build: @@ -513,7 +513,7 @@ def test_create_docker_runner_default_build_profile(): """ Tests the ``conan create . `` """ - client = TestClient(path_with_spaces=False) + client = TestClient() profile_host = textwrap.dedent(f"""\ [settings] From d1bbdf0e0acf800d23f4d92238fcb9380d262023 Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 7 Oct 2024 16:57:41 +0200 Subject: [PATCH 23/39] wip --- test/functional/command/runner_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/command/runner_test.py b/test/functional/command/runner_test.py index 41c7387af74..b385873c74a 100644 --- a/test/functional/command/runner_test.py +++ b/test/functional/command/runner_test.py @@ -362,7 +362,7 @@ def package(self): cmake.install() """) - client = TestClient() + client = TestClient(path_with_spaces=False) client.save({'conanfile.py': conanfile, "CMakeLists.txt": gen_cmakelists(libsources=["hello.cpp"], appsources=["main.cpp"], From 94f6cb08ea873f5f7b8185c2437fd2a6f593c55d Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 7 Oct 2024 17:31:43 +0200 Subject: [PATCH 24/39] minot changes --- .github/workflows/linux-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 4be553120e8..947bfc8ba53 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -67,7 +67,7 @@ jobs: matrix: python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] test-type: ['unittests', 'integration', 'functional'] - name: Python ${{ matrix.python-version }} - ${{ matrix.test-type }} + name: Conan Test Suite - Python ${{ matrix.python-version }} - ${{ matrix.test-type }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -95,11 +95,11 @@ jobs: shell: bash run: | if [ "${{ matrix.test-type }}" == "unittests" ]; then - pytest test/unittests --durations=20 -n 4 -rs + pytest test/unittests --durations=20 -n 4 -m "not docker_runner" elif [ "${{ matrix.test-type }}" == "integration" ]; then - pytest test/integration --durations=20 -n 4 -rs + pytest test/integration --durations=20 -n 4 -m "not docker_runner" elif [ "${{ matrix.test-type }}" == "functional" ]; then - pytest test/functional --durations=20 -n 4 -rs + pytest test/functional --durations=20 -n 4 -m "not docker_runner" fi linux_docker_tests: From 8dd75691f665692e684c0074439596ba154a3d44 Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 7 Oct 2024 17:31:53 +0200 Subject: [PATCH 25/39] fail if all tests are skipped --- test/conftest.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index b9bc9ff5887..15836de91a8 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -359,3 +359,10 @@ def pytest_runtest_setup(item): item.old_environ = dict(os.environ) tools_env_vars['PATH'] = os.pathsep.join(tools_paths + [os.environ["PATH"]]) os.environ.update(tools_env_vars) + + +# fail if all tests are skipped +def pytest_sessionfinish(session, exitstatus): + if session.testscollected > 0 and session.testsfailed == 0 and session.testscollected == session.skipped: + session.exitstatus = 1 + print("All tests were skipped. Failing the test run.") From d6bdec589300e319660eab59216e7d235494f36d Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 7 Oct 2024 17:34:43 +0200 Subject: [PATCH 26/39] minor changes --- .github/workflows/linux-tests.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 947bfc8ba53..5e25e1016de 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -112,13 +112,23 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + run: | + pyenv global ${{ matrix.python-version }} + python --version + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }} + - name: Install dependencies run: | pip install --upgrade pip - pip install -r conans/requirements.txt - pip install -r conans/requirements_dev.txt - pip install -r conans/requirements_server.txt pip install -r conans/requirements_runner.txt + - name: Run tests shell: bash run: | From edf53a6d68e8f1f24a910d678ea8b6b1ff1d2cb9 Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 7 Oct 2024 17:35:01 +0200 Subject: [PATCH 27/39] reduce py versions --- .github/workflows/linux-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 5e25e1016de..52e57a98c50 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -107,7 +107,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] + python-version: ['3.12.3', '3.9.2'] name: Docker Runner Tests - Python ${{ matrix.python-version }} steps: - name: Checkout code From 5702fcb1feaab3b06464fbaa63cfdd155d90fe3b Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 7 Oct 2024 17:39:58 +0200 Subject: [PATCH 28/39] fix python --- .github/workflows/linux-tests.yml | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 52e57a98c50..2c96a76e70f 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -57,7 +57,7 @@ jobs: if: steps.cache_docker_image.outputs.cache-hit != 'true' run: docker push ghcr.io/${{ github.repository_owner }}/conan-tests:latest - linux: + linux_test_suite: needs: build_container runs-on: ubuntu-latest container: @@ -105,28 +105,25 @@ jobs: linux_docker_tests: needs: build_container runs-on: ubuntu-latest + name: Docker Runner Tests strategy: matrix: - python-version: ['3.12.3', '3.9.2'] - name: Docker Runner Tests - Python ${{ matrix.python-version }} + python-version: [3.12, 3.9] steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - run: | - pyenv global ${{ matrix.python-version }} - python --version - - - name: Cache pip - uses: actions/cache@v4 + uses: actions/setup-python@v4 with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }} + python-version: ${{ matrix.python-version }} + + - name: Checkout code + uses: actions/checkout@v4 - name: Install dependencies run: | pip install --upgrade pip + pip install -r conans/requirements.txt + pip install -r conans/requirements_dev.txt + pip install -r conans/requirements_server.txt pip install -r conans/requirements_runner.txt - name: Run tests From 0a4ca8b8667b3b2d448b4c8fb6d31224c1c01b77 Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 7 Oct 2024 17:40:50 +0200 Subject: [PATCH 29/39] does not need container --- .github/workflows/linux-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 2c96a76e70f..3ad5ac7eb1c 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -103,7 +103,6 @@ jobs: fi linux_docker_tests: - needs: build_container runs-on: ubuntu-latest name: Docker Runner Tests strategy: From 9fb65f9d9443c7ad9be23370a06ed8439c0787f8 Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 7 Oct 2024 17:46:25 +0200 Subject: [PATCH 30/39] revert change --- test/conftest.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 15836de91a8..b9bc9ff5887 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -359,10 +359,3 @@ def pytest_runtest_setup(item): item.old_environ = dict(os.environ) tools_env_vars['PATH'] = os.pathsep.join(tools_paths + [os.environ["PATH"]]) os.environ.update(tools_env_vars) - - -# fail if all tests are skipped -def pytest_sessionfinish(session, exitstatus): - if session.testscollected > 0 and session.testsfailed == 0 and session.testscollected == session.skipped: - session.exitstatus = 1 - print("All tests were skipped. Failing the test run.") From 03772b7f96938687daa76d3de058d5348b33413d Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 7 Oct 2024 17:48:05 +0200 Subject: [PATCH 31/39] add name --- .github/workflows/linux-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 3ad5ac7eb1c..fae7d5eebcb 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -16,6 +16,7 @@ concurrency: jobs: build_container: runs-on: ubuntu-latest + name: Build and Publish Docker Image for testing Conan steps: - name: Checkout code uses: actions/checkout@v4 From b00e932e2822ceec1aaed26ab6cfe16e5c7a324b Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 7 Oct 2024 17:58:56 +0200 Subject: [PATCH 32/39] change name --- .github/workflows/linux-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index fae7d5eebcb..773d1929c55 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -68,7 +68,7 @@ jobs: matrix: python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] test-type: ['unittests', 'integration', 'functional'] - name: Conan Test Suite - Python ${{ matrix.python-version }} - ${{ matrix.test-type }} + name: Conan ${{ matrix.test-type }} (${{ matrix.python-version }}) steps: - name: Checkout code uses: actions/checkout@v4 From a03ba51d10e7027be5067fc0fb008cd715b9f28e Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 8 Oct 2024 07:19:52 +0200 Subject: [PATCH 33/39] use v5 --- .github/workflows/linux-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 773d1929c55..be28c7b3283 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -111,7 +111,7 @@ jobs: python-version: [3.12, 3.9] steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From e57778fb694af83bd586d6890c85c7fb128e0dce Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Mon, 14 Oct 2024 07:52:48 +0200 Subject: [PATCH 34/39] Update .github/workflows/linux-tests.yml --- .github/workflows/linux-tests.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index be28c7b3283..0e3c2e96ec8 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -95,12 +95,14 @@ jobs: - name: Run tests shell: bash run: | + NUM_CPUS=$(sysctl -n hw.ncpu) + echo "Number of CPUs available: $NUM_CPUS" if [ "${{ matrix.test-type }}" == "unittests" ]; then - pytest test/unittests --durations=20 -n 4 -m "not docker_runner" + pytest test/unittests --durations=20 -n $NUM_CPUS elif [ "${{ matrix.test-type }}" == "integration" ]; then - pytest test/integration --durations=20 -n 4 -m "not docker_runner" + pytest test/integration --durations=20 -n $NUM_CPUS elif [ "${{ matrix.test-type }}" == "functional" ]; then - pytest test/functional --durations=20 -n 4 -m "not docker_runner" + pytest test/functional --durations=20 -n $NUM_CPUS fi linux_docker_tests: From 6f51d9b16b6fb30d8afc25281ed9a203647d0085 Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Mon, 14 Oct 2024 07:58:38 +0200 Subject: [PATCH 35/39] Update .github/workflows/linux-tests.yml --- .github/workflows/linux-tests.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 0e3c2e96ec8..4acf0f414e9 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -95,14 +95,12 @@ jobs: - name: Run tests shell: bash run: | - NUM_CPUS=$(sysctl -n hw.ncpu) - echo "Number of CPUs available: $NUM_CPUS" if [ "${{ matrix.test-type }}" == "unittests" ]; then - pytest test/unittests --durations=20 -n $NUM_CPUS + pytest test/unittests --durations=20 -n 4 elif [ "${{ matrix.test-type }}" == "integration" ]; then - pytest test/integration --durations=20 -n $NUM_CPUS + pytest test/integration --durations=20 -n 4 elif [ "${{ matrix.test-type }}" == "functional" ]; then - pytest test/functional --durations=20 -n $NUM_CPUS + pytest test/functional --durations=20 -n 4 fi linux_docker_tests: From 597956fe8d68b6b26e6c89778212d3242f5b8320 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 16 Oct 2024 07:53:38 +0200 Subject: [PATCH 36/39] recover bump_dev_version --- .ci/__init__.py | 0 .ci/bump_dev_version.py | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 .ci/__init__.py create mode 100644 .ci/bump_dev_version.py diff --git a/.ci/__init__.py b/.ci/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/.ci/bump_dev_version.py b/.ci/bump_dev_version.py new file mode 100644 index 00000000000..e3f802833ea --- /dev/null +++ b/.ci/bump_dev_version.py @@ -0,0 +1,22 @@ +import os +import time + + +def replace_in_file(file_path, search, replace): + with open(file_path, "r") as handle: + content = handle.read() + if search not in content: + raise Exception("Incorrect development version in conans/__init__.py") + content = content.replace(search, replace) + content = content.encode("utf-8") + with open(file_path, "wb") as handle: + handle.write(content) + +def bump_dev(): + vfile = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../conans/__init__.py") + snapshot = "%s" % int(time.time()) + replace_in_file(vfile, "-dev'", "-dev%s'" % snapshot) + + +if __name__ == "__main__": + bump_dev() From 47fd0582f0b127dd3f4979e50ed0164d698e42d1 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 16 Oct 2024 08:06:52 +0200 Subject: [PATCH 37/39] add deply to pypitest --- .github/workflows/linux-tests.yml | 44 +++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 4acf0f414e9..7c34df6aa66 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -130,3 +130,47 @@ jobs: shell: bash run: | pytest -m docker_runner --durations=20 -rs + + deploy_to_pypi_test: + needs: [build_container, linux_test_suite, linux_docker_tests] + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/develop2' + name: Deploy to TestPyPI + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install twine + + - name: Bump Dev Version + run: | + python .ci/bump_dev_version.py + + - name: Build Package + run: | + python setup.py sdist + + - name: Upload to TestPyPI + env: + TWINE_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }} + run: | + python -m twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/* + + - name: Deploy conan-server to TestPyPI + env: + TWINE_USERNAME: ${{ secrets.TEST_PYPI_SERVER_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_SERVER_PASSWORD }} + run: | + rm -rf dist/ + mv setup_server.py setup.py + python setup.py sdist + python -m twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/* From 2569f5c22b7bf1ca9775eeb1d351341c22b14355 Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 29 Nov 2024 11:21:51 +0100 Subject: [PATCH 38/39] remove fish --- .ci/docker/conan-tests | 2 -- .github/workflows/linux-tests.yml | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.ci/docker/conan-tests b/.ci/docker/conan-tests index d801bd19eff..48d1d8ca375 100644 --- a/.ci/docker/conan-tests +++ b/.ci/docker/conan-tests @@ -61,7 +61,6 @@ RUN apt-get update && \ g++-11 \ clang-14 \ clang++-14 \ - fish \ gcc-arm-linux-gnueabihf \ g++-arm-linux-gnueabihf \ unzip \ @@ -75,7 +74,6 @@ RUN apt-get update && \ # fix: asm/errno.h: No such file or directory ln -s /usr/include/asm-generic/ /usr/include/asm && \ add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ - add-apt-repository -y ppa:fish-shell/release-3 && \ apt-get update && \ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu jammy stable" && \ diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 7c34df6aa66..fb229df9798 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -8,6 +8,7 @@ on: pull_request: branches: - '*' + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} From 301d827b9895ec502aec5b0268d692ddd4a83367 Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 29 Nov 2024 11:24:56 +0100 Subject: [PATCH 39/39] add python 3.13 --- .ci/docker/conan-tests | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/docker/conan-tests b/.ci/docker/conan-tests index 48d1d8ca375..2d176b181fe 100644 --- a/.ci/docker/conan-tests +++ b/.ci/docker/conan-tests @@ -8,6 +8,7 @@ ENV PY36=3.6.15 \ PY38=3.8.6 \ PY39=3.9.2 \ PY312=3.12.3 \ + PY313=3.13.0 \ CMAKE_3_15_7=/usr/share/cmake-3.15.7/bin/cmake \ CMAKE_3_16_9=/usr/share/cmake-3.16.9/bin/cmake \ CMAKE_3_17_5=/usr/share/cmake-3.17.5/bin/cmake \ @@ -93,6 +94,7 @@ RUN curl https://pyenv.run | bash && \ pyenv install $PY38 && \ pyenv install $PY39 && \ pyenv install $PY312 && \ + pyenv install $PY313 && \ pyenv global $PY39 && \ curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python get-pip.py && \