From 44b856268d7c8259e4cce0d5ba560130201610c9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 8 Feb 2024 03:51:45 -0500 Subject: [PATCH 01/10] Test Alpine Linux on CI With only one version of Python, currently 3.11. --- .github/workflows/alpine-test.yml | 56 +++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/alpine-test.yml diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml new file mode 100644 index 000000000..991a8026b --- /dev/null +++ b/.github/workflows/alpine-test.yml @@ -0,0 +1,56 @@ +name: test-alpine + +on: [push, pull_request, workflow_dispatch] + +jobs: + build: + runs-on: ubuntu-latest + + container: + image: alpine:latest + + defaults: + run: + shell: sh -exo pipefail {0} + + steps: + - name: Install Alpine Linux packages + run: | + apk add git git-daemon python3 py3-pip + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Prepare this repo for tests + run: | + ./init-tests-after-clone.sh + + - name: Set git user identity and command aliases for the tests + run: | + git config --global user.email "travis@ci.com" + git config --global user.name "Travis Runner" + # If we rewrite the user's config by accident, we will mess it up + # and cause subsequent tests to fail + cat test/fixtures/.gitconfig >> ~/.gitconfig + + - name: Update PyPA packages + run: | + # Get the latest pip, wheel, and prior to Python 3.12, setuptools. + python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel + + - name: Install project and test dependencies + run: | + pip install ".[test]" + + - name: Show version and platform information + run: | + uname -a + command -v git python + git version + python --version + python -c 'import os, sys; print(f"sys.platform={sys.platform!r}, os.name={os.name!r}")' + + - name: Test with pytest + run: | + pytest --color=yes -p no:sugar --instafail -vv From cefb53e32e814eb120b0e7d0405f7baa7bf8d8f1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 8 Feb 2024 04:12:37 -0500 Subject: [PATCH 02/10] Work around different ownership in container This handles the "dubious ownership" error for the Alpine Linux container using safe.directory as is done in the Cygwin job. Another approach may be to actually use a limited user account in the container, though, and that may be better, since I expect some of the rmtree tests in test_util.py to fail due to the root user being able to perform a delete operation the tests assume cannot be done. --- .github/workflows/alpine-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index 991a8026b..6de18d306 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -22,6 +22,10 @@ jobs: with: fetch-depth: 0 + - name: Special configuration for Alpine Linux git + run: | + git config --global --add safe.directory "$(pwd)" + - name: Prepare this repo for tests run: | ./init-tests-after-clone.sh From a45d0b07509b40ac0701ba72d9d892c46b9ec386 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 13 Feb 2024 18:31:24 -0500 Subject: [PATCH 03/10] Use venv on Alpine Linux To overcome "This environment is externally managed" blocker. --- .github/workflows/alpine-test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index 6de18d306..46b5f139d 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -38,17 +38,24 @@ jobs: # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig + - name: Create Python virtual environment + run: | + python -m venv .venv + - name: Update PyPA packages run: | # Get the latest pip, wheel, and prior to Python 3.12, setuptools. + . .venv/bin/activate python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel - name: Install project and test dependencies run: | + . .venv/bin/activate pip install ".[test]" - name: Show version and platform information run: | + . .venv/bin/activate uname -a command -v git python git version @@ -57,4 +64,5 @@ jobs: - name: Test with pytest run: | + . .venv/bin/activate pytest --color=yes -p no:sugar --instafail -vv From 5de954a35d2953ca873eacaa0ce63525f7134914 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 14 Feb 2024 09:21:38 -0500 Subject: [PATCH 04/10] Run tests as non-root user in Alpine Linux - Add a note to the fixture in test_util.py that its ability to create files where rmtree will fail is contingent on not running as root (since root doesn't need to own a dir to delete from it). - Create a non-root user in the container. Give it the same UID as owns the repository files that are shared with the container. Also create a group with the GID of the repository files that are shared with the container and add the user to the group, though that is less important. Actually creating the user ensures it has a home directory and may help some commands work. Passing `options: --user 1001` under `container:` will not work because, even if we didn't make the user, the `apk add` commands still need to run as root. - Run all commands as the new non-root user, except for the system administration commands that install needed apk packages and set up the new non-root user account. To continue clearly expressing each step separately and have them automatically run in the container, this uses the hacky approach of having the default shell be a "sudo" command that runs the script step with "sh" (and passes the desired shell arguments). - Preserve environment variables that may have been set by or for the GHA runner, in commands that run as the non-root user. That is, pass those through, while still removing/resetting others. If this is not done, then the variables such as `CI`, which the init-tests-after-clone.sh uses to proceed without interactive confirmation, will not be set, and that step will fail. However, I think it is also a good idea to do this, which is why I've included all the relevant variables and not just `CI`. - Now that a non-root user is using "pip", stop using a venv, at least for now. The other test jobs don't use one, since the runners are isolated, and a container on a runner is even more isolated. But it may be best to bring the venv back, maybe even on the other test jobs, or alternatively to use "python -m pip" instead of "pip", to ensure expected version of pip is used. - Don't add safe.directory inside the container, in the hope that this may not be necessary because the cloned repository files should have the same UID (and even GID) as the user using them. But I expect this may need to be put back; it seems to be needed separately from that, as actions/checkout automatically attempts it for the git command it finds and attempts to use. This is not the only approach that could work. Another approach is to make use of the container explicit in each step, rather than using the `container` key. I think that would make the relationship between the commands here and in other test workflows less apparent and make the workflow a bit less clear, but it could also simplify things. A third approach is to create an image with the needed apk packages and user account, which switches to that user, by writing a Dockerfile and building in image, producing it in a previous job and sharing the image with the job that runs the tests so that `container` can still be used. That might be ideal if it could be done with upload-artifact and download-artifact, but I think `container` only supports getting images from registries. --- .github/workflows/alpine-test.yml | 23 ++++++++--------------- test/test_util.py | 4 +++- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index 46b5f139d..cc0db7e20 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -11,21 +11,22 @@ jobs: defaults: run: - shell: sh -exo pipefail {0} + shell: sudo -u runner sh -exo pipefail {0} steps: - - name: Install Alpine Linux packages + - name: Prepare Alpine Linux run: | - apk add git git-daemon python3 py3-pip + apk add sudo git git-daemon python3 py3-pip + echo 'Defaults env_keep += "CI GITHUB_* RUNNER_*"' >/etc/sudoers.d/ci_env + addgroup -g 127 docker + adduser -D -u 1001 runner + adduser runner docker + shell: sh -exo pipefail {0} # Run this as root, not the "runner" user. - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Special configuration for Alpine Linux git - run: | - git config --global --add safe.directory "$(pwd)" - - name: Prepare this repo for tests run: | ./init-tests-after-clone.sh @@ -38,24 +39,17 @@ jobs: # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig - - name: Create Python virtual environment - run: | - python -m venv .venv - - name: Update PyPA packages run: | # Get the latest pip, wheel, and prior to Python 3.12, setuptools. - . .venv/bin/activate python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel - name: Install project and test dependencies run: | - . .venv/bin/activate pip install ".[test]" - name: Show version and platform information run: | - . .venv/bin/activate uname -a command -v git python git version @@ -64,5 +58,4 @@ jobs: - name: Test with pytest run: | - . .venv/bin/activate pytest --color=yes -p no:sugar --instafail -vv diff --git a/test/test_util.py b/test/test_util.py index f3769088c..65f77082d 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -51,7 +51,9 @@ def permission_error_tmpdir(tmp_path): # Set up PermissionError on Windows, where we can't delete read-only files. (td / "x").chmod(stat.S_IRUSR) - # Set up PermissionError on Unix, where we can't delete files in read-only directories. + # Set up PermissionError on Unix, where non-root users can't delete files in + # read-only directories. (Tests that rely on this and assert that rmtree raises + # PermissionError will fail if they are run as root.) td.chmod(stat.S_IRUSR | stat.S_IXUSR) yield td From ab37ae7eb6d23e9d74178915ec9e072f67f0cfd1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 14 Feb 2024 10:33:19 -0500 Subject: [PATCH 05/10] Add back safe.directory step As anticipated, it is still needed. --- .github/workflows/alpine-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index cc0db7e20..fd085109d 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -27,6 +27,10 @@ jobs: with: fetch-depth: 0 + - name: Special configuration for Alpine Linux git + run: | + git config --global --add safe.directory "$(pwd)" + - name: Prepare this repo for tests run: | ./init-tests-after-clone.sh From 46e4234116949b820741322eaedaee4389390934 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 14 Feb 2024 10:37:47 -0500 Subject: [PATCH 06/10] Debug ownership --- .github/workflows/alpine-test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index fd085109d..a1327c1ce 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -27,6 +27,14 @@ jobs: with: fetch-depth: 0 + - name: Debug ownership + run: | + printenv GITHUB_WORKSPACE + printenv RUNNER_WORKSPACE + whoami + id + ls -alR /__w + - name: Special configuration for Alpine Linux git run: | git config --global --add safe.directory "$(pwd)" From 20780cb98db6d30a42ef7432ed065ea92325588e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 14 Feb 2024 13:03:07 -0500 Subject: [PATCH 07/10] Take ownership of cloned repository --- .github/workflows/alpine-test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index a1327c1ce..33f7580bf 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -27,6 +27,11 @@ jobs: with: fetch-depth: 0 + - name: Set workspace ownership + run: | + chown -R runner:docker -- "$GITHUB_WORKSPACE" + shell: sh -exo pipefail {0} # Run this as root, not the "runner" user. + - name: Debug ownership run: | printenv GITHUB_WORKSPACE From b32932f63e013413fb5150d881b67707ee650f22 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 14 Feb 2024 13:59:25 -0500 Subject: [PATCH 08/10] Bring back venv The "error: externally-managed-environment" stoppage occurs even when the Alpine Linux python command is run by a non-root user. --- .github/workflows/alpine-test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index 33f7580bf..cdc1bd5d8 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -56,17 +56,24 @@ jobs: # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig + - name: Create Python virtual environment + run: | + python -m venv .venv + - name: Update PyPA packages run: | # Get the latest pip, wheel, and prior to Python 3.12, setuptools. + . .venv/bin/activate python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel - name: Install project and test dependencies run: | + . .venv/bin/activate pip install ".[test]" - name: Show version and platform information run: | + . .venv/bin/activate uname -a command -v git python git version @@ -75,4 +82,5 @@ jobs: - name: Test with pytest run: | + . .venv/bin/activate pytest --color=yes -p no:sugar --instafail -vv From bad545ab36d37a1f89bb1dbb587130539498bc64 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 14 Feb 2024 14:09:09 -0500 Subject: [PATCH 09/10] Re-remove safe.directory step We chown the workspace, so this shouldn't be needed. This commit also removes the "Debug ownership" step. --- .github/workflows/alpine-test.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index cdc1bd5d8..7634ea1d8 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -32,18 +32,6 @@ jobs: chown -R runner:docker -- "$GITHUB_WORKSPACE" shell: sh -exo pipefail {0} # Run this as root, not the "runner" user. - - name: Debug ownership - run: | - printenv GITHUB_WORKSPACE - printenv RUNNER_WORKSPACE - whoami - id - ls -alR /__w - - - name: Special configuration for Alpine Linux git - run: | - git config --global --add safe.directory "$(pwd)" - - name: Prepare this repo for tests run: | ./init-tests-after-clone.sh From 4b427c994833e1a2781c028ac80a11e1a6a64997 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 14 Feb 2024 15:14:41 -0500 Subject: [PATCH 10/10] Factor venv activation into the venv creation step --- .github/workflows/alpine-test.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index 7634ea1d8..2c1eed391 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -44,24 +44,23 @@ jobs: # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig - - name: Create Python virtual environment + - name: Set up virtualenv run: | python -m venv .venv + . .venv/bin/activate + printf '%s=%s\n' 'PATH' "$PATH" 'VIRTUAL_ENV' "$VIRTUAL_ENV" >>"$GITHUB_ENV" - name: Update PyPA packages run: | # Get the latest pip, wheel, and prior to Python 3.12, setuptools. - . .venv/bin/activate python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel - name: Install project and test dependencies run: | - . .venv/bin/activate pip install ".[test]" - name: Show version and platform information run: | - . .venv/bin/activate uname -a command -v git python git version @@ -70,5 +69,4 @@ jobs: - name: Test with pytest run: | - . .venv/bin/activate pytest --color=yes -p no:sugar --instafail -vv