-
Notifications
You must be signed in to change notification settings - Fork 26
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Co-authored-by: Christopher Harris <[email protected]> Co-authored-by: Vyas Ramasubramani <[email protected]>
- Loading branch information
3 people
authored
Sep 24, 2024
1 parent
ba23d4c
commit b647143
Showing
29 changed files
with
841 additions
and
46 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
23 changes: 23 additions & 0 deletions
23
features/src/utils/opt/devcontainer/bin/creds/s3/generate.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| #! /usr/bin/env bash | ||
|
|
||
| _creds_s3_generate() { | ||
| local -; | ||
| set -euo pipefail; | ||
|
|
||
| # shellcheck disable=SC1091 | ||
| . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-generate'; | ||
|
|
||
| if test -z "${SCCACHE_BUCKET:-}"; then | ||
| exit 1; | ||
| fi | ||
|
|
||
| if test -n "${AWS_ROLE_ARN:-}" && gh nv-gha-aws --help >/dev/null 2>&1; then | ||
| # shellcheck disable=SC1091 | ||
| devcontainer-utils-creds-s3-gh-generate; | ||
| elif test -n "${VAULT_HOST}"; then | ||
| # shellcheck disable=SC1091 | ||
| devcontainer-utils-creds-s3-vault-generate; | ||
| fi | ||
| } | ||
|
|
||
| _creds_s3_generate "$@" <&0; |
68 changes: 68 additions & 0 deletions
68
features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| #! /usr/bin/env bash | ||
|
|
||
| _creds_github_generate() { | ||
| local -; | ||
| set -euo pipefail; | ||
|
|
||
| # shellcheck disable=SC1091 | ||
| . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-vault creds-s3-vault-generate'; | ||
|
|
||
| if test -z "${AWS_ROLE_ARN:-}" \ | ||
| || test -z "${SCCACHE_BUCKET:-}" \ | ||
| || ! gh nv-gha-aws --help >/dev/null 2>&1; then | ||
| exit 1; | ||
| fi | ||
|
|
||
| # Remove existing credentials in case vault declines to issue new ones. | ||
| rm -rf ~/.aws/{stamp,config,credentials}; | ||
|
|
||
| SCCACHE_REGION="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}"; | ||
|
|
||
| devcontainer-utils-creds-s3-persist - <<< \ | ||
| --bucket="${SCCACHE_BUCKET:-}" \ | ||
| --region="${SCCACHE_REGION:-}" ; | ||
| # Initialize the GitHub CLI with the appropriate user scopes | ||
| # shellcheck disable=SC1091 | ||
| . devcontainer-utils-init-github-cli; | ||
| # Check whether the user is in one of the allowed GitHub orgs | ||
| local allowed_orgs="${AWS_GITHUB_ORGS:-${VAULT_GITHUB_ORGS:-nvidia nv-morpheus nv-legate rapids}}"; | ||
| allowed_orgs="${allowed_orgs// /|}"; | ||
| allowed_orgs="${allowed_orgs//;/|}"; | ||
| allowed_orgs="${allowed_orgs//,/|}"; | ||
| local -ra user_orgs="($( \ | ||
| gh api user/orgs --jq '.[].login' \ | ||
| -H "Accept: application/vnd.github+json" \ | ||
| | grep --color=never -iE "(${allowed_orgs})" \ | ||
| ))"; | ||
| if test "${#user_orgs[@]}" -eq 0; then | ||
| exit 1; | ||
| fi | ||
| local org; | ||
| local generated_at; | ||
| local -a nv_gha_aws_args=( | ||
| --profile default | ||
| --output creds-file | ||
| --role-arn "${AWS_ROLE_ARN}" | ||
| --aud "${AWS_AUDIENCE:-sts.amazonaws.com}" | ||
| --duration "${AWS_S3_TTL:-${VAULT_S3_TTL:-43200}}" | ||
| --idp-url "${AWS_IDP_URL:-https://token.gha-runners.nvidia.com}" | ||
| ); | ||
| for org in "${user_orgs[@]}"; do | ||
| generated_at="$(date '+%s')"; | ||
| if gh nv-gha-aws org "${org}" "${nv_gha_aws_args[@]}" >"${HOME}/.aws/credentials" 2>>/var/log/devcontainer-utils/creds-s3.log; then | ||
| if devcontainer-utils-creds-s3-propagate 2>&1 | tee -a /var/log/devcontainer-utils/creds-s3.log; then | ||
| echo "${generated_at}" > ~/.aws/stamp; | ||
| return 0; | ||
| fi | ||
| fi | ||
| done | ||
| } | ||
| _creds_github_generate "$@" <&0; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| #! /usr/bin/env bash | ||
|
|
||
| _s3_cred() { | ||
| sed -n "s/$1=//p" ~/.aws/credentials 2>/dev/null; | ||
| } | ||
|
|
||
| _s3_creds_init() { | ||
| local - | ||
| set -euo pipefail; | ||
|
|
||
| # shellcheck disable=SC1091 | ||
| . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-init'; | ||
|
|
||
| if type sccache >/dev/null 2>&1; then | ||
| if ! grep -qE "^$" <<< "${SCCACHE_BUCKET:-}"; then | ||
| if grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}" \ | ||
| && grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}" ; then | ||
| if ! grep -qE "^$" <<< "${VAULT_HOST:-${AWS_ROLE_ARN:-}}"; then | ||
| # Generate S3 creds if they don't exist (or are expired) | ||
| if devcontainer-utils-creds-s3-test \ | ||
| || devcontainer-utils-creds-s3-generate; then | ||
| # Persist creds in ~/.aws dir | ||
| devcontainer-utils-creds-s3-persist - <<< " \ | ||
| --bucket '${SCCACHE_BUCKET:-}' \ | ||
| --region '${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}' \ | ||
| --aws-access-key-id '$(_s3_cred aws_access_key_id)' \ | ||
| --aws-session-token '$(_s3_cred aws_session_token)' \ | ||
| --aws-secret-access-key '$(_s3_cred aws_secret_access_key)' \ | ||
| "; | ||
| # Install a crontab to refresh the credentials | ||
| devcontainer-utils-creds-s3-schedule; | ||
| else | ||
| devcontainer-utils-creds-s3-persist - <<< "--no-bucket --no-region"; | ||
| fi | ||
| elif devcontainer-utils-creds-s3-test; then | ||
| devcontainer-utils-creds-s3-persist - <<< " \ | ||
| --bucket '${SCCACHE_BUCKET:-}' \ | ||
| --region '${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}' \ | ||
| --aws-access-key-id '$(_s3_cred aws_access_key_id)' \ | ||
| --aws-session-token '$(_s3_cred aws_session_token)' \ | ||
| --aws-secret-access-key '$(_s3_cred aws_secret_access_key)' \ | ||
| "; | ||
| else | ||
| # bucket is inaccessible | ||
| devcontainer-utils-creds-s3-persist - <<< "--no-bucket --no-region"; | ||
| fi | ||
| elif ! devcontainer-utils-creds-s3-propagate; then | ||
| # bucket is inaccessible | ||
| devcontainer-utils-creds-s3-persist <<< "--no-bucket --no-region"; | ||
| fi | ||
| fi | ||
| fi | ||
| } | ||
|
|
||
| _s3_creds_init "$@"; | ||
|
|
||
| # shellcheck disable=SC1090 | ||
| . /etc/profile.d/*-devcontainer-utils.sh; |
Oops, something went wrong.