From 4b00ec567b7d5aea7641ee8d3638dad2cab90ab9 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sun, 8 Nov 2026 23:29:48 -0700 Subject: [PATCH] feat: creds --format env points S3 and STS at the data proxy Co-Authored-By: Claude Opus 5.5 --- README.md | 15 +++++++++-- docs/demos/aws.json | 18 ++++++++++---- docs/demos/aws.svg | 2 +- docs/recorder/guide.py | 6 +++-- src/main.rs | 7 +++--- src/output.rs | 56 ++++++++++++++++++++++++++++++++++++++---- src/service_account.rs | 10 +++----- 7 files changed, 90 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 0a6b8f5..9edc2dc 100644 --- a/README.md +++ b/README.md @@ -250,7 +250,7 @@ The keyring cannot list its entries, so the credentials of other roles stay. You When the credentials expire, `source-coop creds` gets new ones with the session. Run `source-coop auth login` again only if this fails, for example because the refresh token expired or was revoked. -![`aws s3 ls` through `credential_process`, then `creds --format env`](docs/demos/aws.svg) +![`aws s3 ls` through `credential_process`, then with `creds --format env`](docs/demos/aws.svg) ### Output formats @@ -262,12 +262,23 @@ When the credentials expire, `source-coop creds` gets new ones with the session. source-coop creds ``` -**env**: shell `export` statements. They set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` in your shell. +**env**: shell `export` statements. They set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` in your shell, and point S3 and STS at the data proxy (`--proxy-url`), so the AWS CLI and SDKs need no `--endpoint-url`: ```bash eval $(source-coop creds --format env) +aws s3 ls s3://ACCOUNT/PRODUCT/ ``` +After the credentials, it prints: + +```bash +export AWS_REGION=us-west-2 +export AWS_ENDPOINT_URL_S3=https://data.source.coop +export AWS_ENDPOINT_URL_STS=https://data.source.coop/.sts +``` + +Only S3 and STS go to the proxy; other AWS services keep their usual endpoints. The region is there because some SDKs won't run without one: the proxy accepts any. The AWS CLI v2 (2.13 or later), boto3 (1.28 or later), s3fs, obstore and polars read `AWS_ENDPOINT_URL_S3`. GDAL and DuckDB don't, so give them the endpoint their own way. + **aws-credentials**: an INI profile for `~/.aws/credentials`. `--profile` sets the section name. ```bash diff --git a/docs/demos/aws.json b/docs/demos/aws.json index d6a2246..38ce40d 100644 --- a/docs/demos/aws.json +++ b/docs/demos/aws.json @@ -1,21 +1,29 @@ { - "duration": 26.271, + "duration": 34.428, "steps": [ { - "time": 0.755, + "time": 0.761, "label": "$ cat ~/.aws/config" }, { - "time": 4.605, + "time": 4.589, "label": "$ aws s3 ls s3://demo-org/demo-data/ --profile source-coop" }, { - "time": 12.597, + "time": 12.642, "label": "$ aws s3 ls s3://demo-org/demo-data/readings/ --profile source-coop --human-readable" }, { - "time": 21.777, + "time": 21.718, "label": "$ source-coop creds --format env" + }, + { + "time": 26.212, + "label": "$ eval $(source-coop creds --format env)" + }, + { + "time": 29.304, + "label": "$ aws s3 ls s3://demo-org/demo-data/" } ], "keys": [] diff --git a/docs/demos/aws.svg b/docs/demos/aws.svg index 3a6cbfa..463cd10 100644 --- a/docs/demos/aws.svg +++ b/docs/demos/aws.svg @@ -1 +1 @@ -$$cat$cat~$cat~/.aws$cat~/.aws/config[profilesource-coop]credential_process=source-coopcredsendpoint_url=https://data.source.coop$a$aw$aws$awss$awss3$awss3l$awss3ls$awss3lss$awss3lss3$awss3lss3:$awss3lss3:/$awss3lss3://d$awss3lss3://de$awss3lss3://dem$awss3lss3://demo$awss3lss3://demo-$awss3lss3://demo-o$awss3lss3://demo-or$awss3lss3://demo-org$awss3lss3://demo-org/d$awss3lss3://demo-org/de$awss3lss3://demo-org/dem$awss3lss3://demo-org/demo$awss3lss3://demo-org/demo-$awss3lss3://demo-org/demo-d$awss3lss3://demo-org/demo-da$awss3lss3://demo-org/demo-dat$awss3lss3://demo-org/demo-data$awss3lss3://demo-org/demo-data/--profile$awss3lss3://demo-org/demo-data/--profilesource-coopPREreadings/2026-09-1401:30:002048README.md2026-09-1401:30:0018874368stations.parquet$awss3lss3://demo-org/demo-data/readings$awss3lss3://demo-org/demo-data/readings/--profile$awss3lss3://demo-org/demo-data/readings/--profilesource-coop$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--human-readable2026-09-1401:30:0050.0MiB2026-08.parquet2026-09-1401:30:0048.0MiB2026-09.parquet$source-coop$source-coopcreds$source-coopcreds--format$source-coopcreds--formatenvexportAWS_ACCESS_KEY_ID=ASIADEMOEXAMPLE00001exportAWS_SECRET_ACCESS_KEY=demo/secret/not-a-real-keyexportAWS_SESSION_TOKEN=demo-session-token$c$ca$cat~/.$cat~/.a$cat~/.aw$cat~/.aws/c$cat~/.aws/co$cat~/.aws/con$cat~/.aws/conf$cat~/.aws/confi$awss3lss3://demo-org/demo-data/$awss3lss3://demo-org/demo-data/-$awss3lss3://demo-org/demo-data/--$awss3lss3://demo-org/demo-data/--p$awss3lss3://demo-org/demo-data/--pr$awss3lss3://demo-org/demo-data/--pro$awss3lss3://demo-org/demo-data/--prof$awss3lss3://demo-org/demo-data/--profi$awss3lss3://demo-org/demo-data/--profil$awss3lss3://demo-org/demo-data/--profiles$awss3lss3://demo-org/demo-data/--profileso$awss3lss3://demo-org/demo-data/--profilesou$awss3lss3://demo-org/demo-data/--profilesour$awss3lss3://demo-org/demo-data/--profilesourc$awss3lss3://demo-org/demo-data/--profilesource$awss3lss3://demo-org/demo-data/--profilesource-$awss3lss3://demo-org/demo-data/--profilesource-c$awss3lss3://demo-org/demo-data/--profilesource-co$awss3lss3://demo-org/demo-data/--profilesource-coo$awss3lss3://demo-org/demo-data/r$awss3lss3://demo-org/demo-data/re$awss3lss3://demo-org/demo-data/rea$awss3lss3://demo-org/demo-data/read$awss3lss3://demo-org/demo-data/readi$awss3lss3://demo-org/demo-data/readin$awss3lss3://demo-org/demo-data/reading$awss3lss3://demo-org/demo-data/readings/$awss3lss3://demo-org/demo-data/readings/-$awss3lss3://demo-org/demo-data/readings/--$awss3lss3://demo-org/demo-data/readings/--p$awss3lss3://demo-org/demo-data/readings/--pr$awss3lss3://demo-org/demo-data/readings/--pro$awss3lss3://demo-org/demo-data/readings/--prof$awss3lss3://demo-org/demo-data/readings/--profi$awss3lss3://demo-org/demo-data/readings/--profil$awss3lss3://demo-org/demo-data/readings/--profiles$awss3lss3://demo-org/demo-data/readings/--profileso$awss3lss3://demo-org/demo-data/readings/--profilesou$awss3lss3://demo-org/demo-data/readings/--profilesour$awss3lss3://demo-org/demo-data/readings/--profilesourc$awss3lss3://demo-org/demo-data/readings/--profilesource$awss3lss3://demo-org/demo-data/readings/--profilesource-$awss3lss3://demo-org/demo-data/readings/--profilesource-c$awss3lss3://demo-org/demo-data/readings/--profilesource-co$awss3lss3://demo-org/demo-data/readings/--profilesource-coo$awss3lss3://demo-org/demo-data/readings/--profilesource-coop-$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--hu$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--hum$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--huma$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--human$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--human-$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--human-r$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--human-re$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--human-reaaababl$s$so$sou$sour$sourc$source$source-$source-c$source-co$source-coo$source-coopc$source-coopcr$source-coopcre$source-coopcred$source-coopcreds-$source-coopcreds--$source-coopcreds--f$source-coopcreds--fo$source-coopcreds--for$source-coopcreds--form$source-coopcreds--forma$source-coopcreds--formate$source-coopcreds--formaten \ No newline at end of file +$$cat$cat~$cat~/.aws$cat~/.aws/config[profilesource-coop]credential_process=source-coopcredsendpoint_url=https://data.source.coop$a$aw$aws$awss$awss3$awss3l$awss3ls$awss3lss$awss3lss3$awss3lss3:$awss3lss3:/$awss3lss3://d$awss3lss3://de$awss3lss3://dem$awss3lss3://demo$awss3lss3://demo-$awss3lss3://demo-o$awss3lss3://demo-or$awss3lss3://demo-org$awss3lss3://demo-org/d$awss3lss3://demo-org/de$awss3lss3://demo-org/dem$awss3lss3://demo-org/demo$awss3lss3://demo-org/demo-$awss3lss3://demo-org/demo-d$awss3lss3://demo-org/demo-da$awss3lss3://demo-org/demo-dat$awss3lss3://demo-org/demo-data$awss3lss3://demo-org/demo-data/$awss3lss3://demo-org/demo-data/--profile$awss3lss3://demo-org/demo-data/--profilesource-coopPREreadings/2026-09-1401:30:002048README.md2026-09-1401:30:0018874368stations.parquet$awss3lss3://demo-org/demo-data/readings$awss3lss3://demo-org/demo-data/readings/--profile$awss3lss3://demo-org/demo-data/readings/--profilesource-coop$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--human-readable2026-09-1401:30:0050.0MiB2026-08.parquet2026-09-1401:30:0048.0MiB2026-09.parquet$source-coop$source-coopcreds$source-coopcreds--format$source-coopcreds--formatenvexportAWS_ACCESS_KEY_ID=ASIADEMOEXAMPLE00001exportAWS_SECRET_ACCESS_KEY=demo/secret/not-a-real-keyexportAWS_SESSION_TOKEN=demo-session-tokenexportAWS_REGION=us-west-2exportAWS_ENDPOINT_URL_S3=https://data.source.coopexportAWS_ENDPOINT_URL_STS=https://data.source.coop/.sts$eval$eval$(source-coop$eval$(source-coopcreds$eval$(source-coopcreds--format$eval$(source-coopcreds--formatenv)$c$ca$cat~/.$cat~/.a$cat~/.aw$cat~/.aws/c$cat~/.aws/co$cat~/.aws/con$cat~/.aws/conf$cat~/.aws/confi$awss3lss3://demo-org/demo-data/-$awss3lss3://demo-org/demo-data/--$awss3lss3://demo-org/demo-data/--p$awss3lss3://demo-org/demo-data/--pr$awss3lss3://demo-org/demo-data/--pro$awss3lss3://demo-org/demo-data/--prof$awss3lss3://demo-org/demo-data/--profi$awss3lss3://demo-org/demo-data/--profil$awss3lss3://demo-org/demo-data/--profiles$awss3lss3://demo-org/demo-data/--profileso$awss3lss3://demo-org/demo-data/--profilesou$awss3lss3://demo-org/demo-data/--profilesour$awss3lss3://demo-org/demo-data/--profilesourc$awss3lss3://demo-org/demo-data/--profilesource$awss3lss3://demo-org/demo-data/--profilesource-$awss3lss3://demo-org/demo-data/--profilesource-c$awss3lss3://demo-org/demo-data/--profilesource-co$awss3lss3://demo-org/demo-data/--profilesource-coo$awss3lss3://demo-org/demo-data/r$awss3lss3://demo-org/demo-data/re$awss3lss3://demo-org/demo-data/rea$awss3lss3://demo-org/demo-data/read$awss3lss3://demo-org/demo-data/readi$awss3lss3://demo-org/demo-data/readin$awss3lss3://demo-org/demo-data/reading$awss3lss3://demo-org/demo-data/readings/$awss3lss3://demo-org/demo-data/readings/-$awss3lss3://demo-org/demo-data/readings/--$awss3lss3://demo-org/demo-data/readings/--p$awss3lss3://demo-org/demo-data/readings/--pr$awss3lss3://demo-org/demo-data/readings/--pro$awss3lss3://demo-org/demo-data/readings/--prof$awss3lss3://demo-org/demo-data/readings/--profi$awss3lss3://demo-org/demo-data/readings/--profil$awss3lss3://demo-org/demo-data/readings/--profiles$awss3lss3://demo-org/demo-data/readings/--profileso$awss3lss3://demo-org/demo-data/readings/--profilesou$awss3lss3://demo-org/demo-data/readings/--profilesour$awss3lss3://demo-org/demo-data/readings/--profilesourc$awss3lss3://demo-org/demo-data/readings/--profilesource$awss3lss3://demo-org/demo-data/readings/--profilesource-$awss3lss3://demo-org/demo-data/readings/--profilesource-c$awss3lss3://demo-org/demo-data/readings/--profilesource-co$awss3lss3://demo-org/demo-data/readings/--profilesource-coo$awss3lss3://demo-org/demo-data/readings/--profilesource-coop-$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--hu$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--hum$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--huma$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--human$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--human-$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--human-r$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--human-re$awss3lss3://demo-org/demo-data/readings/--profilesource-coop--human-reaaababl$s$so$sou$sour$sourc$source$source-$source-c$source-co$source-coo$source-coopc$source-coopcr$source-coopcre$source-coopcred$source-coopcreds-$source-coopcreds--$source-coopcreds--f$source-coopcreds--fo$source-coopcreds--for$source-coopcreds--form$source-coopcreds--forma$source-coopcreds--formate$source-coopcreds--formaten$e$ev$eva$eval$$eval$($eval$(s$eval$(so$eval$(sou$eval$(sour$eval$(sourc$eval$(source$eval$(source-$eval$(source-c$eval$(source-co$eval$(source-coo$eval$(source-coopc$eval$(source-coopcr$eval$(source-coopcre$eval$(source-coopcred$eval$(source-coopcreds-$eval$(source-coopcreds--$eval$(source-coopcreds--f$eval$(source-coopcreds--fo$eval$(source-coopcreds--for$eval$(source-coopcreds--form$eval$(source-coopcreds--forma$eval$(source-coopcreds--formate$eval$(source-coopcreds--formaten$eval$(source-coopcreds--formatenv \ No newline at end of file diff --git a/docs/recorder/guide.py b/docs/recorder/guide.py index d6a6c46..f803f08 100644 --- a/docs/recorder/guide.py +++ b/docs/recorder/guide.py @@ -79,7 +79,9 @@ def aws(s): s.run("cat ~/.aws/config"); s.prompt(); s.pause(2) s.run(f"aws s3 ls s3://{ORG}/demo-data/ --profile source-coop"); s.prompt(); s.pause(1.5) s.run(f"aws s3 ls s3://{ORG}/demo-data/readings/ --profile source-coop --human-readable"); s.prompt(); s.pause(1.5) - s.run("source-coop creds --format env") + s.run("source-coop creds --format env"); s.prompt(); s.pause(3) + s.run("eval $(source-coop creds --format env)"); s.prompt() + s.run(f"aws s3 ls s3://{ORG}/demo-data/") # no --profile or --endpoint-url def prompts(s): @@ -396,7 +398,7 @@ def service_account_usage(s): "auth-status": ("`auth status`, then `--json`", auth_status), "auth-token": ("`auth token`: the session's access token, for `curl`", auth_token), "auth-logout": ("`auth logout --json`, then `auth status` and `creds` signed out", auth_logout), - "aws": ("`aws s3 ls` through `credential_process`, then `creds --format env`", aws), + "aws": ("`aws s3 ls` through `credential_process`, then with `creds --format env`", aws), "prompts": ("Prompts: a refused ID asked again, a draft kept after a failure, then Esc", prompts), "completion": ("Shell completion: TAB lists products and service accounts from the API", completion), "account-list": ("`account list`, `view`, then `list --json`", account_list), diff --git a/src/main.rs b/src/main.rs index 50f1c28..96bfdb4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -386,7 +386,8 @@ struct ServiceAccountArgs { enum OutputFormat { /// AWS credential_process JSON format CredentialProcess, - /// Shell export statements + /// Shell export statements: the credentials, AWS_REGION, and the + /// --proxy-url as the S3 and STS endpoints Env, /// AWS credentials file (INI) profile AwsCredentials, @@ -925,7 +926,7 @@ async fn run_login(args: LoginArgs, verbose: bool) -> Result<(), String> { eprintln!("Skipping credential cache (--no-cache)"); match args.output.format { OutputFormat::CredentialProcess => output::print_credential_process(&creds), - OutputFormat::Env => output::print_env(&creds), + OutputFormat::Env => output::print_env(&creds, &args.env.proxy_url), OutputFormat::AwsCredentials => { output::print_aws_credentials(&creds, &args.output.profile) } @@ -1014,7 +1015,7 @@ async fn run_creds(args: CredsArgs, verbose: bool) -> Result<(), String> { match args.output.format { OutputFormat::CredentialProcess => output::print_credential_process(&entry.creds), - OutputFormat::Env => output::print_env(&entry.creds), + OutputFormat::Env => output::print_env(&entry.creds, &args.env.proxy_url), OutputFormat::AwsCredentials => { output::print_aws_credentials(&entry.creds, &args.output.profile) } diff --git a/src/output.rs b/src/output.rs index e69cb6e..3815450 100644 --- a/src/output.rs +++ b/src/output.rs @@ -21,11 +21,33 @@ pub fn print_credential_process(creds: &Credentials) { println!("{}", serde_json::to_string_pretty(&json).unwrap()); } -/// Print credentials as shell export statements. -pub fn print_env(creds: &Credentials) { - println!("export AWS_ACCESS_KEY_ID={}", creds.access_key_id); - println!("export AWS_SECRET_ACCESS_KEY={}", creds.secret_access_key); - println!("export AWS_SESSION_TOKEN={}", creds.session_token); +/// The exports that point a stock AWS SDK or the AWS CLI at the data proxy: +/// S3 and STS only (a global `AWS_ENDPOINT_URL` would send IAM and every +/// other service there too), and a region, which the Rust and Go SDKs won't +/// run without and the proxy ignores. +pub fn endpoint_env(proxy: &str) -> [String; 3] { + let proxy = proxy.trim_end_matches('/'); + [ + "export AWS_REGION=us-west-2".to_string(), + format!("export AWS_ENDPOINT_URL_S3={proxy}"), + format!("export AWS_ENDPOINT_URL_STS={proxy}/.sts"), + ] +} + +/// Credentials as shell export statements, with the proxy's endpoints. +fn env(creds: &Credentials, proxy: &str) -> String { + let mut lines = vec![ + format!("export AWS_ACCESS_KEY_ID={}", creds.access_key_id), + format!("export AWS_SECRET_ACCESS_KEY={}", creds.secret_access_key), + format!("export AWS_SESSION_TOKEN={}", creds.session_token), + ]; + lines.extend(endpoint_env(proxy)); + lines.join("\n") +} + +/// Print credentials as shell export statements, with the proxy's endpoints. +pub fn print_env(creds: &Credentials, proxy: &str) { + println!("{}", env(creds, proxy)); } /// Print credentials as an AWS credentials-file (INI) profile, @@ -37,3 +59,27 @@ pub fn print_aws_credentials(creds: &Credentials, profile: &str) { println!("aws_secret_access_key = {}", creds.secret_access_key); println!("aws_session_token = {}", creds.session_token); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn env_exports_credentials_and_the_proxy_s_endpoints() { + let creds = Credentials { + access_key_id: "AKID".into(), + secret_access_key: "SECRET".into(), + session_token: "TOKEN".into(), + expiration: "2026-07-13T12:00:00Z".into(), + }; + let want = "export AWS_ACCESS_KEY_ID=AKID\n\ + export AWS_SECRET_ACCESS_KEY=SECRET\n\ + export AWS_SESSION_TOKEN=TOKEN\n\ + export AWS_REGION=us-west-2\n\ + export AWS_ENDPOINT_URL_S3=https://data.source.coop\n\ + export AWS_ENDPOINT_URL_STS=https://data.source.coop/.sts"; + assert_eq!(env(&creds, "https://data.source.coop"), want); + // A trailing slash on --proxy-url doesn't double up before /.sts. + assert_eq!(env(&creds, "https://data.source.coop/"), want); + } +} diff --git a/src/service_account.rs b/src/service_account.rs index 14e3734..6e29e85 100644 --- a/src/service_account.rs +++ b/src/service_account.rs @@ -2759,14 +2759,12 @@ fn github_workflow(proxy: &str, id: &str, subject: &str) -> String { /// What a stock AWS SDK or the AWS CLI needs to sign in with an API key saved /// to a file: the web UI's `apiKeyEnvironment`. fn api_key_environment(proxy: &str, id: &str) -> String { - [ - "export AWS_REGION=us-west-2".to_string(), - format!("export AWS_ENDPOINT_URL_S3={proxy}"), - format!("export AWS_ENDPOINT_URL_STS={proxy}/.sts"), + let mut lines = crate::output::endpoint_env(proxy).to_vec(); + lines.extend([ format!("export AWS_ROLE_ARN=arn:aws:iam::{id}:role/FullAccess"), "export AWS_WEB_IDENTITY_TOKEN_FILE=/path/to/the/saved/key".into(), - ] - .join("\n") + ]); + lines.join("\n") } #[cfg(test)]