Forward-merge branch-24.10 into branch-24.12

.devcontainer/rapids.Dockerfile

@@ -47,7 +47,8 @@ ENV PYTHONDONTWRITEBYTECODE="1"
 
 ENV SCCACHE_REGION="us-east-2"
 ENV SCCACHE_BUCKET="rapids-sccache-devs"
-ENV VAULT_HOST="https://vault.ops.k8s.rapids.ai"
+ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs"
+
 ENV HISTFILE="/home/coder/.cache/._bash_history"
 
 ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/jitify_cache"

.github/actions/build-and-test-feature/action.yml

@@ -6,6 +6,7 @@ inputs:
   args: {type: string, required: true}
   gh_token: {type: string, defaut: '', required: false}
   vault_host: {type: string, defaut: '', required: false}
+  aws_role_arn: {type: string, defaut: '', required: false}
   rw_sccache_bucket: {type: string, defaut: '', required: false}
   rw_sccache_region: {type: string, defaut: '', required: false}
 
@@ -30,5 +31,6 @@ runs:
         VAULT_S3_TTL: "900" # 15 minutes
         gh_token: "${{ inputs.gh_token }}"
         vault_host: "${{ inputs.vault_host }}"
+        aws_role_arn: "${{ inputs.aws_role_arn }}"
         rw_sccache_bucket: "${{ inputs.rw_sccache_bucket }}"
         rw_sccache_region: "${{ inputs.rw_sccache_region }}"

.github/workflows/build-and-test-feature.yml

@@ -38,5 +38,6 @@ jobs:
           args: "${{ inputs.args }}"
           gh_token: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN }}"
           vault_host: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'https://vault.ops.k8s.rapids.ai' || '' }}"
+          aws_role_arn: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs' || '' }}"
           rw_sccache_bucket: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'rapids-sccache-devs' || '' }}"
           rw_sccache_region: "${{ vars.AWS_REGION }}"

features/src/rapids-build-utils/devcontainer-feature.json

@@ -1,7 +1,7 @@
 {
   "name": "NVIDIA RAPIDS devcontainer build utilities",
   "id": "rapids-build-utils",
-  "version": "24.10.9",
+  "version": "24.10.10",
   "description": "A feature to install the RAPIDS devcontainer build utilities",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"

features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.clone.tmpl.sh

@@ -51,7 +51,6 @@ clone_${NAME}() {
         --branch "${branch}"                    \
         --ssh-url "${ssh_url}"                  \
         --https-url "${https_url}"              \
-        --recurse-submodules                    \
         -j ${n_jobs:-$(nproc --all --ignore=1)} \
         -c checkout.defaultRemote=upstream      \
         "${OPTS[@]}"                            \

features/src/utils/devcontainer-feature.json

@@ -1,7 +1,7 @@
 {
   "name": "devcontainer-utils",
   "id": "utils",
-  "version": "24.10.3",
+  "version": "24.10.5",
   "description": "A feature to install RAPIDS devcontainer utility scripts",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"

features/src/utils/install.sh

@@ -67,9 +67,9 @@ chmod u+s "$(realpath -m "$(which cron)")";
 
 # shellcheck disable=SC2174
 mkdir -m 0775 -p /var/log/devcontainer-utils;
-touch /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
-chmod 0664 /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
-chgrp crontab /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
+touch /var/log/devcontainer-utils/creds-s3.log;
+chmod 0664 /var/log/devcontainer-utils/creds-s3.log;
+chgrp crontab /var/log/devcontainer-utils/creds-s3.log;
 
 # Install Devcontainer utility scripts to /opt/devcontainer
 cp -ar ./opt/devcontainer /opt/;
@@ -79,6 +79,15 @@ declare -a commands_and_sources=(
     "parse-args                         parse-args.sh"
     "parse-args-from-docstring          parse-args-from-docstring.sh"
     "bash-completion.tmpl               bash/completion.tmpl.sh"
+    "creds-s3-init                      creds/s3/init.sh"
+    "creds-s3-generate                  creds/s3/generate.sh"
+    "creds-s3-persist                   creds/s3/persist.sh"
+    "creds-s3-propagate                 creds/s3/propagate.sh"
+    "creds-s3-schedule                  creds/s3/schedule.sh"
+    "creds-s3-test                      creds/s3/test.sh"
+    "creds-s3-gh-generate               creds/s3/gh/generate.sh"
+    "creds-s3-vault-generate            creds/s3/vault/generate.sh"
+    "creds-s3-vault-github              creds/s3/vault/github.sh"
     "generate-bash-completion           bash/generate-bash-completion.sh"
     "shell-is-interactive               shell-is-interactive.sh"
     "post-create-command                post-create-command.sh"
@@ -107,10 +116,10 @@ declare -a commands_and_sources=(
 
 # Install alternatives
 for entry in "${commands_and_sources[@]}"; do
-    declare -a pair=(${entry});
+    declare -a pair="(${entry})";
     declare cmd="devcontainer-utils-${pair[0]}";
     declare src="/opt/devcontainer/bin/${pair[1]}";
-    update-alternatives --install /usr/bin/${cmd} ${cmd} ${src} 0;
+    update-alternatives --install "/usr/bin/${cmd}" "${cmd}" "${src}" 0;
 done
 
 declare -a commands="($(for pair in "${commands_and_sources[@]}"; do cut -d' ' -f1 <<< "${pair}"; done))";
@@ -182,6 +191,30 @@ find_non_root_user;
 
 if test -n "${USERNAME-}"; then
     USERHOME="$(bash -c "echo ~${USERNAME-}")";
+
+    if type gh >/dev/null 2>&1; then
+        mkdir -p -m 0755                                         \
+            "$USERHOME/.local"                                   \
+            "$USERHOME/.local/share"                             \
+            "$USERHOME/.local/share/gh"                          \
+            "$USERHOME/.local/share/gh/extensions"               \
+            "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws" \
+            ;
+        NV_GHA_AWS_VERSION=latest
+        find_version_from_git_tags NV_GHA_AWS_VERSION https://github.com/nv-gha-runners/gh-nv-gha-aws;
+        wget --no-hsts -q -O "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws" \
+            "https://github.com/nv-gha-runners/gh-nv-gha-aws/releases/download/v${NV_GHA_AWS_VERSION}/gh-nv-gha-aws_v${NV_GHA_AWS_VERSION}_linux-$(dpkg --print-architecture | awk -F'-' '{print $NF}')";
+        chmod 0755 "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws";
+        cat <<EOF >"$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/manifest.yml"
+owner: nv-gha-runners
+name: gh-nv-gha-aws
+host: github.com
+tag: v${NV_GHA_AWS_VERSION}
+ispinned: false
+path: $USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws
+EOF
+    fi
+
     # Add user to the crontab group
     usermod -aG crontab "${USERNAME}";
     # Allow user to edit the crontab

features/src/utils/opt/devcontainer/bin/creds/s3/generate.sh

@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+_creds_s3_generate() {
+    local -;
+    set -euo pipefail;
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-generate';
+
+    if test -z "${SCCACHE_BUCKET:-}"; then
+        exit 1;
+    fi
+
+    if test -n "${AWS_ROLE_ARN:-}" && gh nv-gha-aws --help >/dev/null 2>&1; then
+        # shellcheck disable=SC1091
+        devcontainer-utils-creds-s3-gh-generate;
+    elif test -n "${VAULT_HOST}"; then
+        # shellcheck disable=SC1091
+        devcontainer-utils-creds-s3-vault-generate;
+    fi
+}
+
+_creds_s3_generate "$@" <&0;

features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh

@@ -0,0 +1,68 @@
+#! /usr/bin/env bash
+
+_creds_github_generate() {
+    local -;
+    set -euo pipefail;
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-vault creds-s3-vault-generate';
+
+    if test -z "${AWS_ROLE_ARN:-}" \
+    || test -z "${SCCACHE_BUCKET:-}" \
+    || ! gh nv-gha-aws --help >/dev/null 2>&1; then
+        exit 1;
+    fi
+
+    # Remove existing credentials in case vault declines to issue new ones.
+    rm -rf ~/.aws/{stamp,config,credentials};
+
+    SCCACHE_REGION="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}";
+
+    devcontainer-utils-creds-s3-persist - <<< \
+        --bucket="${SCCACHE_BUCKET:-}"        \
+        --region="${SCCACHE_REGION:-}"        ;
+
+    # Initialize the GitHub CLI with the appropriate user scopes
+    # shellcheck disable=SC1091
+    . devcontainer-utils-init-github-cli;
+
+    # Check whether the user is in one of the allowed GitHub orgs
+    local allowed_orgs="${AWS_GITHUB_ORGS:-${VAULT_GITHUB_ORGS:-nvidia nv-morpheus nv-legate rapids}}";
+    allowed_orgs="${allowed_orgs// /|}";
+    allowed_orgs="${allowed_orgs//;/|}";
+    allowed_orgs="${allowed_orgs//,/|}";
+
+    local -ra user_orgs="($(                         \
+        gh api user/orgs --jq '.[].login'            \
+            -H "Accept: application/vnd.github+json" \
+        | grep --color=never -iE "(${allowed_orgs})" \
+    ))";
+
+    if test "${#user_orgs[@]}" -eq 0; then
+        exit 1;
+    fi
+
+    local org;
+    local generated_at;
+    local -a nv_gha_aws_args=(
+        --profile default
+        --output creds-file
+        --role-arn "${AWS_ROLE_ARN}"
+        --aud "${AWS_AUDIENCE:-sts.amazonaws.com}"
+        --duration "${AWS_S3_TTL:-${VAULT_S3_TTL:-43200}}"
+        --idp-url "${AWS_IDP_URL:-https://token.gha-runners.nvidia.com}"
+    );
+
+    for org in "${user_orgs[@]}"; do
+        generated_at="$(date '+%s')";
+        if gh nv-gha-aws org "${org}" "${nv_gha_aws_args[@]}" >"${HOME}/.aws/credentials" 2>>/var/log/devcontainer-utils/creds-s3.log; then
+            if devcontainer-utils-creds-s3-propagate 2>&1 | tee -a /var/log/devcontainer-utils/creds-s3.log; then
+                echo "${generated_at}" > ~/.aws/stamp;
+                return 0;
+            fi
+        fi
+    done
+
+}
+
+_creds_github_generate "$@" <&0;

features/src/utils/opt/devcontainer/bin/creds/s3/init.sh

@@ -0,0 +1,58 @@
+#! /usr/bin/env bash
+
+_s3_cred() {
+    sed -n "s/$1=//p" ~/.aws/credentials 2>/dev/null;
+}
+
+_s3_creds_init() {
+    local -
+    set -euo pipefail;
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-init';
+
+    if type sccache >/dev/null 2>&1; then
+        if ! grep -qE "^$" <<< "${SCCACHE_BUCKET:-}"; then
+            if grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}"     \
+            && grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}" ; then
+                if ! grep -qE "^$" <<< "${VAULT_HOST:-${AWS_ROLE_ARN:-}}"; then
+                    # Generate S3 creds if they don't exist (or are expired)
+                    if devcontainer-utils-creds-s3-test \
+                    || devcontainer-utils-creds-s3-generate; then
+                        # Persist creds in ~/.aws dir
+                        devcontainer-utils-creds-s3-persist - <<< "                     \
+                            --bucket '${SCCACHE_BUCKET:-}'                              \
+                            --region '${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}'       \
+                            --aws-access-key-id '$(_s3_cred aws_access_key_id)'         \
+                            --aws-session-token '$(_s3_cred aws_session_token)'         \
+                            --aws-secret-access-key '$(_s3_cred aws_secret_access_key)' \
+                        ";
+                        # Install a crontab to refresh the credentials
+                        devcontainer-utils-creds-s3-schedule;
+                    else
+                        devcontainer-utils-creds-s3-persist - <<< "--no-bucket --no-region";
+                    fi
+                elif devcontainer-utils-creds-s3-test; then
+                    devcontainer-utils-creds-s3-persist - <<< "                     \
+                        --bucket '${SCCACHE_BUCKET:-}'                              \
+                        --region '${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}'       \
+                        --aws-access-key-id '$(_s3_cred aws_access_key_id)'         \
+                        --aws-session-token '$(_s3_cred aws_session_token)'         \
+                        --aws-secret-access-key '$(_s3_cred aws_secret_access_key)' \
+                    ";
+                else
+                    # bucket is inaccessible
+                    devcontainer-utils-creds-s3-persist - <<< "--no-bucket --no-region";
+                fi
+            elif ! devcontainer-utils-creds-s3-propagate; then
+                # bucket is inaccessible
+                devcontainer-utils-creds-s3-persist <<< "--no-bucket --no-region";
+            fi
+        fi
+    fi
+}
+
+_s3_creds_init "$@";
+
+# shellcheck disable=SC1090
+. /etc/profile.d/*-devcontainer-utils.sh;

features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+# Usage:
+#  devcontainer-utils-creds-s3-persist [OPTION]...
+#
+# Set, unset, or reset the S3 bucket, region, and credentials in the environment.
+#
+# Boolean options:
+#  -h,--help                      print this text
+#  --no-bucket                    Unset the $SCCACHE_BUCKET environment variable for all shells.
+#                                 (default: false)
+#  --no-region                    Unset the $SCCACHE_REGION environment variable for all shells.
+#                                 (default: false)
+#
+# Options that require values:
+#  --stamp <stamp>                Timestamp when the S3 credentials were generated.
+#                                 (default: none)
+#  --bucket <bucket>              Set the $SCCACHE_BUCKET environment variable for all shells to <bucket> and persist in ~/.aws/config.
+#                                 (default: none)
+#  --region <region>              Set the $SCCACHE_REGION environment variable for all shells to <region> and persist in ~/.aws/config.
+#                                 (default: none)
+#  --aws-access-key-id <id>       Set the $AWS_ACCESS_KEY_ID environment variable for all shells to <id> and persist in ~/.aws/credentials.
+#                                 (default: none)
+#  --aws-session-token <token>    Set the $AWS_SESSION_TOKEN environment variable for all shells to <token> and persist in ~/.aws/credentials.
+#                                 (default: none)
+#  --aws-secret-access-key <key>  Set the $AWS_SECRET_ACCESS_KEY environment variable for all shells to <key> and persist in ~/.aws/credentials.
+#                                 (default: none)
+
+# shellcheck disable=SC1091
+. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/../../update-envvars.sh";
+
+_creds_s3_persist() {
+    local -;
+    set -euo pipefail;
+
+    eval "$(devcontainer-utils-parse-args "$0" "$@" <&0)";
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-persist';
+
+    # Reset envvars
+    reset_envvar "SCCACHE_BUCKET";
+    reset_envvar "SCCACHE_REGION";
+    reset_envvar "AWS_ACCESS_KEY_ID";
+    reset_envvar "AWS_SESSION_TOKEN";
+    reset_envvar "AWS_SECRET_ACCESS_KEY";
+
+    mkdir -p ~/.aws;
+    rm -f ~/.aws/{config,credentials};
+
+    if test -n "${stamp:-}"; then
+        echo "${stamp:-}" > ~/.aws/stamp;
+    fi
+
+    if ! grep -qE "^$" <<< "${no_bucket-}"; then
+        unset_envvar "SCCACHE_BUCKET";
+    elif ! grep -qE "^$" <<< "${bucket:-}"; then
+        export_envvar "SCCACHE_BUCKET" "${bucket}";
+        cat <<________EOF >> ~/.aws/config
+bucket=${bucket:-}
+________EOF
+    fi
+
+    if ! grep -qE "^$" <<< "${no_region-}"; then
+        unset_envvar "SCCACHE_REGION";
+    elif ! grep -qE "^$" <<< "${region:-}"; then
+        export_envvar "SCCACHE_REGION" "${region}";
+        cat <<________EOF >> ~/.aws/config
+region=${region:-}
+________EOF
+    fi
+
+    if test -f ~/.aws/config; then
+        cat <<________EOF > ~/.aws/config2 && mv ~/.aws/config{2,}
+[default]
+$(cat ~/.aws/config)
+________EOF
+    fi
+
+    if ! grep -qE "^$" <<< "${aws_access_key_id:-}"; then
+        cat <<________EOF >> ~/.aws/credentials
+aws_access_key_id=${aws_access_key_id}
+________EOF
+    fi
+
+    if ! grep -qE "^$" <<< "${aws_secret_access_key:-}"; then
+        cat <<________EOF >> ~/.aws/credentials
+aws_secret_access_key=${aws_secret_access_key}
+________EOF
+    fi
+
+    if ! grep -qE "^$" <<< "${aws_session_token:-}"; then
+        cat <<________EOF >> ~/.aws/credentials
+aws_session_token=${aws_session_token}
+________EOF
+    fi
+
+    if test -f ~/.aws/credentials; then
+        cat <<________EOF > ~/.aws/credentials2 && mv ~/.aws/credentials{2,}
+[default]
+$(cat ~/.aws/credentials)
+________EOF
+        chmod 0600 ~/.aws/credentials;
+    fi
+}
+
+if [ "$(basename "${BASH_SOURCE[${#BASH_SOURCE[@]}-1]}")" = devcontainer-utils-creds-s3-persist ]; then
+    _creds_s3_persist "$@" <&0;
+fi