Merge bitcoin/bitcoin#33002: ci: Only pass documented env vars

3333d3f ci: Only pass documented env vars (MarcoFalke)

Pull request description:

  The CI currently inherits almost all env vars from the host. This was problematic in the past and causing non-determinism, e.g. the fix in commit fa12558. It is still problematic today, see e.g. bitcoin/bitcoin#31349 (comment), or bitcoin/bitcoin#32935

  This fixes bitcoin/bitcoin#32935 by only passing env vars documented in `./ci/test/00_setup_env.sh`.

  Implementation-wise, instead of cramming the python code into the `python -c ""` statement, just start a fresh py file, which is easier to handle.

ACKs for top commit:
  willcl-ark:
    ACK 3333d3f

Tree-SHA512: f922e481a844128d7fbf773563278a3992c178ead60a3050eceb9ded2aad979afc815a5cbdb9f68494493c5d8d942cdd1111c21e32a5746d19505b87745cb84a

Author: fanquake

ci/test/02_run_container.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/license/mit/.
+
+import os
+import shlex
+import subprocess
+import sys
+
+
+def run(cmd, **kwargs):
+    print("+ " + shlex.join(cmd), flush=True)
+    try:
+        return subprocess.run(cmd, check=True, **kwargs)
+    except Exception as e:
+        sys.exit(e)
+
+
+def main():
+    print("Export only allowed settings:")
+    settings = run(
+        ["bash", "-c", "grep export ./ci/test/00_setup_env*.sh"],
+        stdout=subprocess.PIPE,
+        text=True,
+        encoding="utf8",
+    ).stdout.splitlines()
+    settings = set(l.split("=")[0].split("export ")[1] for l in settings)
+    # Add this one manually, because it is the only one set inside the
+    # container that also allows external overwrites
+    settings.add("BASE_BUILD_DIR")
+
+    # Append $USER to /tmp/env to support multi-user systems and $CONTAINER_NAME
+    # to allow support starting multiple runs simultaneously by the same user.
+    env_file = "/tmp/env-{u}-{c}".format(
+        u=os.getenv("USER"),
+        c=os.getenv("CONTAINER_NAME"),
+    )
+    with open(env_file, "w", encoding="utf8") as file:
+        for k, v in os.environ.items():
+            if k in settings:
+                file.write(f"{k}={v}\n")
+    run(["cat", env_file])
+
+    run(["./ci/test/02_run_container.sh"])  # run the remainder
+
+
+if __name__ == "__main__":
+    main()

ci/test/02_run_container.sh

@@ -10,10 +10,6 @@ export CI_IMAGE_LABEL="bitcoin-ci-test"
 set -o errexit -o pipefail -o xtrace
 
 if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
-  # Export all env vars to avoid missing some.
-  # Though, exclude those with newlines to avoid parsing problems.
-  python3 -c 'import os; [print(f"{key}={value}") for key, value in os.environ.items() if "\n" not in value and "HOME" != key and "PATH" != key and "USER" != key]' | tee "/tmp/env-$USER-$CONTAINER_NAME"
-
   # Env vars during the build can not be changed. For example, a modified
   # $MAKEJOBS is ignored in the build process. Use --cpuset-cpus as an
   # approximation to respect $MAKEJOBS somewhat, if cpuset is available.
@@ -118,8 +114,6 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
   # When detecting podman-docker, `--external` should be added.
   docker image prune --force --filter "label=$CI_IMAGE_LABEL"
 
-  # Append $USER to /tmp/env to support multi-user systems and $CONTAINER_NAME
-  # to allow support starting multiple runs simultaneously by the same user.
   # shellcheck disable=SC2086
   CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \
                   --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \

ci/test_run_all.sh

@@ -8,4 +8,4 @@ export LC_ALL=C.UTF-8
 
 set -o errexit; source ./ci/test/00_setup_env.sh
 set -o errexit
-"./ci/test/02_run_container.sh"
+"./ci/test/02_run_container.py"