From d7871aeb92b96aaf8a88921b6f3f4abed33576d3 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 13 Oct 2026 22:22:11 -0700 Subject: [PATCH 1/4] feat: exec, to run a command with credentials and the proxy endpoint source-coop exec -- aws s3 ls s3://acct/prod/ runs any S3 tool as the signed-in user, as aws-vault exec does: the role's current credentials (cached, or minted as creds would) go into the command's environment, with the proxy as AWS_ENDPOINT_URL(_S3) and S3_ENDPOINT_URL, a region if none is set, and AWS_PROFILE removed. On Unix the CLI becomes the command, so its exit code and signals reach the shell directly. creds and exec share current_creds(), which mint() now takes a role and proxy for rather than CredsArgs. Co-Authored-By: Claude Opus 5.5 --- src/cache.rs | 26 ++++++---- src/exec.rs | 113 +++++++++++++++++++++++++++++++++++++++++ src/main.rs | 140 +++++++++++++++++++++++++++++++++++++++------------ 3 files changed, 237 insertions(+), 42 deletions(-) create mode 100644 src/exec.rs diff --git a/src/cache.rs b/src/cache.rs index 4514ec1..1909e4e 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -386,8 +386,8 @@ pub(crate) fn write_new_file(path: &Path, text: &str) -> Result Ok(path.display().to_string()) } -/// Check if credentials are expired or will expire within a 60-second buffer. -pub fn is_expired(creds: &Credentials) -> Result { +/// Whether `creds` run out within `left` from now (or already have). +pub fn expires_within(creds: &Credentials, left: chrono::Duration) -> Result { let expiration = chrono::DateTime::parse_from_rfc3339(&creds.expiration).map_err(|e| { format!( "Failed to parse expiration timestamp '{}': {e}", @@ -395,10 +395,7 @@ pub fn is_expired(creds: &Credentials) -> Result { ) })?; - let now = Utc::now(); - let buffer = chrono::Duration::seconds(60); - - Ok(expiration <= now + buffer) + Ok(expiration <= Utc::now() + left) } #[cfg(test)] @@ -517,14 +514,14 @@ mod tests { fn expired_future_date() { let future = (Utc::now() + chrono::Duration::hours(1)).to_rfc3339(); let creds = sample_creds(&future); - assert!(!is_expired(&creds).unwrap()); + assert!(!expires_within(&creds, chrono::Duration::seconds(60)).unwrap()); } #[test] fn expired_past_date() { let past = (Utc::now() - chrono::Duration::hours(1)).to_rfc3339(); let creds = sample_creds(&past); - assert!(is_expired(&creds).unwrap()); + assert!(expires_within(&creds, chrono::Duration::seconds(60)).unwrap()); } #[test] @@ -532,13 +529,22 @@ mod tests { // 30 seconds from now is within the 60s buffer let near_future = (Utc::now() + chrono::Duration::seconds(30)).to_rfc3339(); let creds = sample_creds(&near_future); - assert!(is_expired(&creds).unwrap()); + assert!(expires_within(&creds, chrono::Duration::seconds(60)).unwrap()); } #[test] fn expired_invalid_timestamp() { let creds = sample_creds("not-a-timestamp"); - assert!(is_expired(&creds).is_err()); + assert!(expires_within(&creds, chrono::Duration::seconds(60)).is_err()); + } + + #[test] + fn expires_within_a_longer_margin() { + // Fine for `creds`, too short for `exec`'s 15 minutes. + let soon = (Utc::now() + chrono::Duration::minutes(10)).to_rfc3339(); + let creds = sample_creds(&soon); + assert!(!expires_within(&creds, chrono::Duration::seconds(60)).unwrap()); + assert!(expires_within(&creds, chrono::Duration::minutes(15)).unwrap()); } #[test] diff --git a/src/exec.rs b/src/exec.rs new file mode 100644 index 0000000..17c89c6 --- /dev/null +++ b/src/exec.rs @@ -0,0 +1,113 @@ +//! `source-coop exec`: run any S3 tool (the AWS CLI, s5cmd, rclone, an SDK +//! script) as the signed-in user, with credentials and the data proxy's +//! endpoint in its environment, as `aws-vault exec` does for AWS. + +use crate::sts::Credentials; +use std::process::Command; + +/// The environment `command` runs with: the credentials, where they expire, +/// and the proxy as the S3 endpoint under the names the common tools read. +/// A region is only given when none is set, as the proxy ignores it but +/// signing needs one. +pub fn vars(creds: &Credentials, proxy_url: &str, region_set: bool) -> Vec<(&'static str, String)> { + let endpoint = proxy_url.trim_end_matches('/').to_string(); + let mut vars = vec![ + ("AWS_ACCESS_KEY_ID", creds.access_key_id.clone()), + ("AWS_SECRET_ACCESS_KEY", creds.secret_access_key.clone()), + ("AWS_SESSION_TOKEN", creds.session_token.clone()), + ("AWS_CREDENTIAL_EXPIRATION", creds.expiration.clone()), + // The AWS CLI (v2.13+) and SDKs, for S3 alone: other services (STS, + // KMS) a script calls keep their own endpoints. + ("AWS_ENDPOINT_URL_S3", endpoint.clone()), + // s5cmd. + ("S3_ENDPOINT_URL", endpoint), + ]; + if !region_set { + vars.push(("AWS_REGION", "us-east-1".into())); + vars.push(("AWS_DEFAULT_REGION", "us-east-1".into())); + } + vars +} + +/// Run `command` with `vars` added to the environment, and leave with its +/// exit code. On Unix this process becomes the command, so signals (Ctrl+C) +/// and the exit status reach the shell as if it had been run directly. +pub fn run(command: &[String], vars: Vec<(&'static str, String)>) -> Result<(), String> { + let (program, args) = command.split_first().ok_or("Name a command to run.")?; + let mut cmd = Command::new(program); + cmd.args(args).envs(vars); + // A profile could hold other credentials or an endpoint; the env vars + // already win, but naming one invites confusion over which applied. And + // tools told to ignore configured endpoints would send the credentials + // to AWS itself rather than the proxy. + for name in [ + "AWS_PROFILE", + "AWS_DEFAULT_PROFILE", + "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", + ] { + cmd.env_remove(name); + } + let failed = |e: std::io::Error| format!("Couldn't run {program}: {e}"); + + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + // Only returns if the command couldn't be started. + Err(failed(cmd.exec())) + } + #[cfg(not(unix))] + { + let status = cmd.status().map_err(failed)?; + std::process::exit(status.code().unwrap_or(1)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn creds() -> Credentials { + Credentials { + access_key_id: "AKID".into(), + secret_access_key: "SECRET".into(), + session_token: "TOKEN".into(), + expiration: "2026-10-14T13:00:00Z".into(), + } + } + + #[test] + fn the_command_gets_credentials_and_the_proxy_as_its_endpoint() { + let vars = vars(&creds(), "https://data.staging.source.coop/", false); + let get = |k: &str| { + vars.iter() + .find(|(name, _)| *name == k) + .map(|(_, v)| v.as_str()) + }; + assert_eq!(get("AWS_ACCESS_KEY_ID"), Some("AKID")); + assert_eq!(get("AWS_SESSION_TOKEN"), Some("TOKEN")); + assert_eq!( + get("AWS_CREDENTIAL_EXPIRATION"), + Some("2026-10-14T13:00:00Z") + ); + for name in ["AWS_ENDPOINT_URL_S3", "S3_ENDPOINT_URL"] { + assert_eq!( + get(name), + Some("https://data.staging.source.coop"), + "{name}" + ); + } + assert_eq!(get("AWS_ENDPOINT_URL"), None, "only S3 goes to the proxy"); + assert_eq!(get("AWS_REGION"), Some("us-east-1")); + } + + #[test] + fn a_region_already_set_is_kept() { + let vars = vars(&creds(), "https://data.source.coop", true); + assert!(!vars.iter().any(|(name, _)| name.contains("REGION"))); + } + + #[test] + fn a_missing_command_is_an_error() { + assert_eq!(run(&[], vec![]), Err("Name a command to run.".to_string())); + } +} diff --git a/src/main.rs b/src/main.rs index 50f1c28..5cdea2a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ mod api_cmd; mod cache; mod complete; mod draft; +mod exec; mod failure; mod member; mod oidc; @@ -127,6 +128,23 @@ enum Commands { /// commands don't cover #[command(after_help = style::help(&API_HELP.replace("{api}", defaults::API_URL)))] Api(ApiCommandArgs), + /// Run a command (the AWS CLI, s5cmd, an SDK script) with credentials and + /// the data proxy as its S3 endpoint in its environment + /// (e.g. `source-coop exec -- aws s3 ls s3://my-org/my-product/`) + Exec(ExecArgs), +} + +#[derive(Parser)] +struct ExecArgs { + #[command(flatten)] + role: RoleArgs, + + #[command(flatten)] + env: EnvArgs, + + /// The command to run, with its arguments, after `--` + #[arg(required = true, trailing_var_arg = true, allow_hyphen_values = true)] + command: Vec, } /// `source-coop api --help` after the flags: examples, then where to find @@ -532,6 +550,11 @@ async fn run() { fail(e, false); } } + Commands::Exec(args) => { + if let Err(e) = run_exec(args, verbose).await { + fail(e, false); + } + } } } @@ -994,24 +1017,48 @@ async fn run_login(args: LoginArgs, verbose: bool) -> Result<(), String> { Ok(()) } -async fn run_creds(args: CredsArgs, verbose: bool) -> Result<(), String> { - let env = cache::env(&args.env.proxy_url)?; - let env = env.as_deref(); - let cached = cache::read_credentials(env, &args.role.role_arn)?; - let entry = match cached { - Some(entry) if !cache::is_expired(&entry.creds)? => entry, - _ => { - // Re-check under the lock: another process may have minted already. - let _lock = cache::lock(env, &args.role.role_arn)?; - match cache::read_credentials(env, &args.role.role_arn)? { - Some(entry) if !cache::is_expired(&entry.creds)? => entry, - stale => mint(&args, env, stale, verbose).await.map_err(|e| { - format!("Couldn't get new credentials ({e}). Run 'source-coop auth login'.") - })?, - } - } - }; +/// Run the command with the role's credentials. They're fetched once, so +/// cached ones are only reused with `EXEC_MIN_LIFETIME` left; a command +/// running past their expiry (an hour unless `login --duration` said +/// otherwise) needs running again. +async fn run_exec(args: ExecArgs, verbose: bool) -> Result<(), String> { + let entry = current_creds( + &args.role.role_arn, + &args.env.proxy_url, + EXEC_MIN_LIFETIME, + verbose, + ) + .await?; + // Set but empty is unset, as signing would fail on an empty region. + let region_set = ["AWS_REGION", "AWS_DEFAULT_REGION"] + .iter() + .any(|v| std::env::var_os(v).is_some_and(|r| !r.is_empty())); + let vars = exec::vars(&entry.creds, &args.env.proxy_url, region_set); + if verbose { + eprintln!( + "[verbose] Running {:?} against {} with credentials expiring {}", + args.command, args.env.proxy_url, entry.creds.expiration + ); + } + exec::run(&args.command, vars) +} + +/// How long cached credentials must still last for `exec` to hand them to a +/// command, which can't ask for new ones as `credential_process` can. +const EXEC_MIN_LIFETIME: chrono::Duration = chrono::Duration::minutes(15); +/// How long cached credentials must still last for `creds`: its callers ask +/// again when they run out. +const CREDS_MIN_LIFETIME: chrono::Duration = chrono::Duration::seconds(60); + +async fn run_creds(args: CredsArgs, verbose: bool) -> Result<(), String> { + let entry = current_creds( + &args.role.role_arn, + &args.env.proxy_url, + CREDS_MIN_LIFETIME, + verbose, + ) + .await?; match args.output.format { OutputFormat::CredentialProcess => output::print_credential_process(&entry.creds), OutputFormat::Env => output::print_env(&entry.creds), @@ -1022,37 +1069,66 @@ async fn run_creds(args: CredsArgs, verbose: bool) -> Result<(), String> { Ok(()) } +/// The role's credentials at `proxy_url`'s environment: the cached ones while +/// they have `min_left` to run, otherwise new ones, minted once however many +/// ask at a time. +async fn current_creds( + role_arn: &str, + proxy_url: &str, + min_left: chrono::Duration, + verbose: bool, +) -> Result { + let env = cache::env(proxy_url)?; + let env = env.as_deref(); + let fresh = |entry: &cache::CacheEntry| -> Result { + Ok(!cache::expires_within(&entry.creds, min_left)?) + }; + match cache::read_credentials(env, role_arn)? { + Some(entry) if fresh(&entry)? => return Ok(entry), + _ => {} + } + // Re-check under the lock: another process may have minted already. + let _lock = cache::lock(env, role_arn)?; + match cache::read_credentials(env, role_arn)? { + Some(entry) if fresh(&entry)? => Ok(entry), + stale => mint(role_arn, proxy_url, env, stale, verbose) + .await + .map_err(|e| match e.as_str() { + NOT_LOGGED_IN => "Not signed in. Run 'source-coop auth login'.".to_string(), + _ => format!("Couldn't get new credentials ({e}). Run 'source-coop auth login'."), + }), + } +} + +/// `mint`'s error when there's no login session to mint with. +const NOT_LOGGED_IN: &str = "not logged in"; + /// New credentials for the role, cached before they're returned. An entry /// from before the session was shared refreshes with its own token; anything /// else exchanges the session's ID token, refreshing the session if need be. async fn mint( - args: &CredsArgs, + role_arn: &str, + proxy_url: &str, env: Option<&str>, stale: Option, verbose: bool, ) -> Result { - let save = - |e: &cache::CacheEntry| cache::write_credentials(env, &args.role.role_arn, e).map(drop); + let save = |e: &cache::CacheEntry| cache::write_credentials(env, role_arn, e).map(drop); match stale { Some(entry) if entry.sts.is_none() && entry.refresh.is_some() => { - refresh(&args.role.role_arn, entry, verbose, save).await + refresh(role_arn, entry, verbose, save).await } stale => { let sts = stale.and_then(|e| e.sts).unwrap_or(cache::StsSettings { - proxy_url: args.env.proxy_url.clone(), + proxy_url: proxy_url.to_string(), duration: None, }); let id_token = session::token(env, session::Want::Id, verbose) .await? - .ok_or("not logged in")?; - let creds = sts::assume_role( - &sts.proxy_url, - &args.role.role_arn, - &id_token, - sts.duration, - verbose, - ) - .await?; + .ok_or(NOT_LOGGED_IN)?; + let creds = + sts::assume_role(&sts.proxy_url, role_arn, &id_token, sts.duration, verbose) + .await?; let entry = cache::CacheEntry { creds, sts: Some(sts), @@ -1770,7 +1846,7 @@ mod tests { let entry = result.unwrap(); assert_eq!(entry.creds.access_key_id, "NEWKEY"); - assert!(!cache::is_expired(&entry.creds).unwrap()); + assert!(!cache::expires_within(&entry.creds, chrono::Duration::seconds(60)).unwrap()); assert_eq!(entry.refresh.unwrap().refresh_token, "new-rt"); // Rotated token persisted first (with old creds), then the new creds. assert_eq!( From 2c9293ae0dd466fbdc9b1315cdbbd6fd9a3a0499 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Thu, 15 Oct 2026 09:34:03 -0700 Subject: [PATCH 2/4] feat: exec also points GDAL and object_store at the proxy GDAL's /vsis3/ (QGIS, rasterio, geopandas) reads no AWS endpoint setting, and object_store before 0.13.2 reads AWS_ENDPOINT rather than AWS_ENDPOINT_URL_S3. Neither can use a credential_process profile on common versions, so they're what exec is most needed for. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01C2VyGzXgtd9L6f6Rs9xDSZ --- src/exec.rs | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/exec.rs b/src/exec.rs index 17c89c6..95baaef 100644 --- a/src/exec.rs +++ b/src/exec.rs @@ -20,8 +20,30 @@ pub fn vars(creds: &Credentials, proxy_url: &str, region_set: bool) -> Vec<(&'st // KMS) a script calls keep their own endpoints. ("AWS_ENDPOINT_URL_S3", endpoint.clone()), // s5cmd. - ("S3_ENDPOINT_URL", endpoint), + ("S3_ENDPOINT_URL", endpoint.clone()), + // Rust `object_store` (obstore, Polars without boto3) before 0.13.2, + // which doesn't read AWS_ENDPOINT_URL_S3; the AWS SDKs ignore it. + ("AWS_ENDPOINT", endpoint), ]; + if let Ok(url) = url::Url::parse(proxy_url) { + let http = url.scheme() == "http"; + if http { + // `object_store` refuses plain HTTP (a local proxy) without it. + vars.push(("AWS_ALLOW_HTTP", "true".into())); + } + // GDAL's /vsis3/ (QGIS, rasterio, geopandas), which reads no AWS + // endpoint setting: a host without the scheme for GDAL before 3.11, + // and paths rather than bucket subdomains. + if let Some(host) = url.host_str() { + let host = match url.port() { + Some(port) => format!("{host}:{port}"), + None => host.to_string(), + }; + vars.push(("AWS_S3_ENDPOINT", host)); + vars.push(("AWS_HTTPS", if http { "NO" } else { "YES" }.into())); + vars.push(("AWS_VIRTUAL_HOSTING", "FALSE".into())); + } + } if !region_set { vars.push(("AWS_REGION", "us-east-1".into())); vars.push(("AWS_DEFAULT_REGION", "us-east-1".into())); @@ -97,9 +119,26 @@ mod tests { ); } assert_eq!(get("AWS_ENDPOINT_URL"), None, "only S3 goes to the proxy"); + assert_eq!( + get("AWS_ENDPOINT"), + Some("https://data.staging.source.coop") + ); + assert_eq!(get("AWS_S3_ENDPOINT"), Some("data.staging.source.coop")); + assert_eq!(get("AWS_HTTPS"), Some("YES")); + assert_eq!(get("AWS_VIRTUAL_HOSTING"), Some("FALSE")); + assert_eq!(get("AWS_ALLOW_HTTP"), None); assert_eq!(get("AWS_REGION"), Some("us-east-1")); } + #[test] + fn a_local_proxy_over_http_keeps_its_port() { + let vars = vars(&creds(), "http://localhost:9000", false); + let get = |k: &str| vars.iter().find(|(n, _)| *n == k).map(|(_, v)| v.as_str()); + assert_eq!(get("AWS_S3_ENDPOINT"), Some("localhost:9000")); + assert_eq!(get("AWS_HTTPS"), Some("NO")); + assert_eq!(get("AWS_ALLOW_HTTP"), Some("true")); + } + #[test] fn a_region_already_set_is_kept() { let vars = vars(&creds(), "https://data.source.coop", true); From 5041a04768dfcc2cc3c2b079411cbc780e4dc92d Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Sun, 8 Nov 2026 23:40:14 -0700 Subject: [PATCH 3/4] feat: exec runs the command unsigned when nobody is signed in Signed out, `exec` now runs the command anyway, unsigned, so public products can be read. It tells GDAL (AWS_NO_SIGN_REQUEST) and object_store (AWS_SKIP_SIGNATURE) not to sign, hides every other AWS key from the command (the proxy refuses keys it doesn't know, even for public data), and says on stderr that it's running unsigned, with the --no-sign-request the AWS CLI and s5cmd need. A session that can't be renewed is still an error. The region given when none is set is now us-west-2, as in `service-account usage`. exec gets help examples, a line in --help's summary for scripts, and its section in "Reading data with AWS tools", with two README demos. The demo stub refuses unsigned listings of products that aren't public, and renewing a session. Co-Authored-By: Claude Opus 5.5 --- README.md | 61 ++++++- docs/demos/exec-signed-out.json | 18 ++ docs/demos/exec-signed-out.svg | 1 + docs/demos/exec.json | 18 ++ docs/demos/exec.svg | 1 + docs/recorder/fixtures.json | 18 ++ docs/recorder/guide.py | 15 ++ docs/recorder/stub.py | 36 +++- src/exec.rs | 286 +++++++++++++++++++++++++------- src/main.rs | 142 ++++++++++++++-- src/sts.rs | 2 +- 11 files changed, 513 insertions(+), 85 deletions(-) create mode 100644 docs/demos/exec-signed-out.json create mode 100644 docs/demos/exec-signed-out.svg create mode 100644 docs/demos/exec.json create mode 100644 docs/demos/exec.svg diff --git a/README.md b/README.md index 0a6b8f5..3ea15d4 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ The command line for [Source Cooperative](https://source.coop). Use it to: -- **Get data credentials.** Sign in through your browser once. AWS tools then get temporary S3 credentials for the Source Cooperative data proxy, and the CLI renews them when they expire. +- **Get data credentials.** Sign in through your browser once. AWS tools then get temporary S3 credentials for the Source Cooperative data proxy, and the CLI renews them when they expire. `exec` runs any S3 tool against the proxy, signed in or not. - **Manage source.coop.** List, view, create, edit and delete products, members and service accounts, and create and edit organizations. The source.coop API applies the same rules as the web UI. All commands work the same way: @@ -31,6 +31,9 @@ source-coop product create # after you add the AWS profile (below) aws s3 ls s3://my-org/my-data --profile source-coop + +# or with no profile at all +source-coop exec -- aws s3 ls s3://my-org/my-data/ ``` `source-coop --help` lists the commands. `source-coop --help` lists the flags of a command, with examples. @@ -285,6 +288,50 @@ aws_session_token = ... > [!TIP] > These credentials are temporary. When they expire, run the command again. Each run adds a new section, and AWS uses the last one, so remove old sections occasionally. We recommend `credential_process` in `~/.aws/config` instead. +### Run a tool with `exec` + +`exec` runs a command with the data proxy as its S3 endpoint and your credentials in its environment, as `aws-vault exec` does for AWS. It needs no profile, and works for tools that can't use one: GDAL (QGIS, rasterio, geopandas), Rust's `object_store` (obstore, Polars) and s5cmd. + +```bash +source-coop exec -- aws s3 ls s3://my-org/my-data/ +source-coop exec -- aws s3 cp s3://my-org/my-data/data.parquet . +source-coop exec -- s5cmd cp 's3://my-org/my-data/*.parquet' ./local/ +source-coop exec -- gdalinfo /vsis3/my-org/my-data/image.tif +source-coop exec -- python script.py +``` + +Everything after `--` is the command, flags included. The command's exit code is passed on. + +![`exec`: the AWS CLI and s5cmd with your credentials and the proxy, no profile](docs/demos/exec.svg) + +The command gets: + +- **Credentials:** `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` and `AWS_CREDENTIAL_EXPIRATION`. +- **The proxy as its S3 endpoint**, under each tool's name for it: `AWS_ENDPOINT_URL_S3` (the AWS CLI v2.13+ and SDKs), `S3_ENDPOINT_URL` (s5cmd), `AWS_ENDPOINT` (`object_store`, with `AWS_ALLOW_HTTP` for a plain-HTTP proxy), and `AWS_S3_ENDPOINT`, `AWS_HTTPS` and `AWS_VIRTUAL_HOSTING=FALSE` (GDAL's `/vsis3/`). Only S3 goes to the proxy: calls a script makes to other AWS services keep their own endpoints. +- **A region**, `AWS_REGION` and `AWS_DEFAULT_REGION` set to `us-west-2`, only if neither is set already, as signing needs one. + +`AWS_PROFILE`, `AWS_DEFAULT_PROFILE` and `AWS_IGNORE_CONFIGURED_ENDPOINT_URLS` are removed, so a profile can't lend other credentials or send these anywhere but the proxy. `--role-arn` and `--proxy-url` (or `SOURCE_ROLE_ARN` and `SOURCE_PROXY_URL`) select the role and environment, as for `creds`. + +The credentials are fetched once, when the command starts. Cached ones are used if they have at least 15 minutes left. Otherwise `exec` gets new ones. A command still running when they expire (an hour after they're fetched, unless you signed in with `--duration`) must be run again. For long jobs, use the [`credential_process` profile](#reading-data-with-aws-tools), which renews them. + +**Not signed in,** `exec` still runs the command, unsigned, so it can read public products. It says so on stderr, and: + +- tells GDAL and `object_store` not to sign (`AWS_NO_SIGN_REQUEST=YES`, `AWS_SKIP_SIGNATURE=true`); +- hides every other AWS key from the command, because the proxy refuses keys it doesn't know, even for public data: it removes `AWS_ACCESS_KEY_ID` and the other credential variables, points `AWS_SHARED_CREDENTIALS_FILE` and `AWS_CONFIG_FILE` at an empty file, and sets `AWS_EC2_METADATA_DISABLED=true`. + +The AWS CLI and s5cmd sign every request unless given `--no-sign-request`, and without credentials they stop with `Unable to locate credentials` or `NoCredentialProviders`. `exec` suggests the flag when it's missing: + +```bash +source-coop exec -- aws --no-sign-request s3 ls s3://cholmes/admin-boundaries/ +source-coop exec -- s5cmd --no-sign-request ls s3://cholmes/admin-boundaries/ +``` + +A product that isn't public is refused with `AccessDenied`, as it is to anyone not signed in. + +![`exec` signed out: unsigned, a public product listed, a restricted one refused](docs/demos/exec-signed-out.svg) + +If you are signed in but the session can't be renewed (the refresh token expired or was revoked), `exec` stops with the error instead of running unsigned. Run `source-coop auth login` again, or `source-coop auth logout` to read public data unsigned. + ### Multiple roles > [!WARNING] @@ -335,6 +382,8 @@ The data proxy accepts a write when all of these are true. Otherwise it answers Name the object, or end the path with `/`. `aws s3 cp data.parquet s3://my-org/my-data` has no object key, and the proxy refuses it with `Missing object key`. +`exec` writes too, signed in: `source-coop exec -- aws s3 sync ./output s3://my-org/my-data/output/`. + To upload from a GitHub Actions workflow or a script, sign in as a [service account](#service-accounts) with write access to the product (see [Access to products](#access-to-products)). `service-account usage` prints a workflow that signs in (see [Use a service account](#use-a-service-account)). Replace its last step, `aws s3 ls`, with your `aws s3 sync`. @@ -1019,7 +1068,7 @@ With `--json`: - Field names are the API's. Times are RFC 3339. - New fields can be added. Fields are not renamed or removed. -These commands do not accept `--json`, because their output is already for machines: `creds` (`--format`), `auth token` (a token), `api` (the API's response), `completions` (a shell script) and `service-account usage` (a workflow or shell exports). `auth login` does not accept it either, because it uses the browser. Use `auth status --json` after it. +These commands do not accept `--json`, because their output is already for machines: `creds` (`--format`), `auth token` (a token), `api` (the API's response), `completions` (a shell script), `service-account usage` (a workflow or shell exports) and `exec` (its command's output, and its command's exit code). `auth login` does not accept it either, because it uses the browser. Use `auth status --json` after it. With `--json`, an error is also JSON, on stderr. If the error came from the API, it contains the data of the API's [RFC 9457](https://www.rfc-editor.org/rfc/rfc9457) problem: @@ -1062,7 +1111,7 @@ curl -H "Authorization: Bearer $(source-coop auth token)" https://source.coop/ap ![`auth token`: the session's access token, for `curl`](docs/demos/auth-token.svg) -To send a request through the CLI, see [Any API request](#any-api-request). +To send a request through the CLI, see [Any API request](#any-api-request). To read or write data, run the S3 tool through [`exec`](#run-a-tool-with-exec): `source-coop exec -- aws s3 ls s3://ACCOUNT/PRODUCT/`. ## Any API request @@ -1109,7 +1158,7 @@ The CLI has no configuration file. Flags and environment variables set all value | `SOURCE_PROXY_URL` | `--proxy-url` | `https://data.source.coop` | The environment: the data proxy, and the session and credentials to use | | `SOURCE_API_URL` | `--api-url` | `https://source.coop` | The source.coop site whose API (`/api/v1`) commands use | | `SOURCE_TOKEN` | | | A bearer token for the API, used instead of the session's | -| `SOURCE_ROLE_ARN` | `--role-arn` | `_default` | The role for `creds`, `auth login`, `status` and `logout` | +| `SOURCE_ROLE_ARN` | `--role-arn` | `_default` | The role for `creds`, `exec`, `auth login`, `status` and `logout` | | `SOURCE_OIDC_ISSUER` | `--issuer` | `https://auth.source.coop` | Where `auth login` signs in | | `SOURCE_OIDC_CLIENT_ID` | `--client-id` | source.coop's | The OAuth2 client for `auth login` | | `SOURCE_API_AUDIENCE` | `--audience` | `https://source.coop` | The audience of the API access token that `auth login` requests | @@ -1203,7 +1252,9 @@ The session is in `source-coop/session.json`. Other environments use the same la **`Your login has expired. Run 'source-coop auth login'.`** The session's refresh token expired or was revoked. Sign in again. `source-coop auth status` estimates when it expires. -**`Couldn't get new credentials (…). Run 'source-coop auth login'.`** `creds` could not renew a role's credentials. AWS tools show this message as their own error. The reason is in the brackets. Usually the session has expired, so sign in again. +**`Couldn't get new credentials (…). Run 'source-coop auth login'.`** `creds` or `exec` could not renew a role's credentials. AWS tools show this message as their own error. The reason is in the brackets. Usually the session has expired, so sign in again. + +**`Unable to locate credentials` or `NoCredentialProviders` under `exec`.** You are not signed in, so `exec` runs the command unsigned, and the AWS CLI and s5cmd then need `--no-sign-request`. Add it to read public data, or run `source-coop auth login`. See [Run a tool with `exec`](#run-a-tool-with-exec). **macOS asks for your keychain password.** macOS asks the first time a new build reads the `source-coop-cli` keychain items, for example after an upgrade. Select **Always Allow** so that it does not ask again until the next upgrade. diff --git a/docs/demos/exec-signed-out.json b/docs/demos/exec-signed-out.json new file mode 100644 index 0000000..b8ed010 --- /dev/null +++ b/docs/demos/exec-signed-out.json @@ -0,0 +1,18 @@ +{ + "duration": 22.863, + "steps": [ + { + "time": 0.752, + "label": "$ source-coop exec -- aws s3 ls s3://partner-org/ocean-buoys/" + }, + { + "time": 7.231, + "label": "$ source-coop exec -- aws --no-sign-request s3 ls s3://partner-org/ocean-buoys/" + }, + { + "time": 14.67, + "label": "$ source-coop exec -- aws --no-sign-request s3 ls s3://demo-org/landsat-mirror/; echo \"exit $?\"" + } + ], + "keys": [] +} \ No newline at end of file diff --git a/docs/demos/exec-signed-out.svg b/docs/demos/exec-signed-out.svg new file mode 100644 index 0000000..898656b --- /dev/null +++ b/docs/demos/exec-signed-out.svg @@ -0,0 +1 @@ +$$s$so$sou$sour$sourc$source$source-$source-c$source-co$source-coo$source-coop$source-coope$source-coopex$source-coopexe$source-coopexec$source-coopexec-$source-coopexec--$source-coopexec--a$source-coopexec--aw$source-coopexec--aws$source-coopexec--awss3$source-coopexec--awss3ls$source-coopexec--awss3lss3:$source-coopexec--awss3lss3:/$source-coopexec--awss3lss3://partner-org$source-coopexec--awss3lss3://partner-org/ocean-buoys$source-coopexec--awss3lss3://partner-org/ocean-buoys/Notsignedin,sorunningunsigned:onlypublicdatacanberead.awsneeds--no-sign-requesttorununsigned:source-coopexec--aws--no-sign-requests3lss3://partner-org/ocean-buoys/Unabletolocatecredentials.Youcanconfigurecredentialsbyrunning"awsconfigure".$source-coopexec--aws-$source-coopexec--aws--$source-coopexec--aws--n$source-coopexec--aws--no$source-coopexec--aws--no-$source-coopexec--aws--no-s$source-coopexec--aws--no-si$source-coopexec--aws--no-sig$source-coopexec--aws--no-sign$source-coopexec--aws--no-sign-$source-coopexec--aws--no-sign-r$source-coopexec--aws--no-sign-re$source-coopexec--aws--no-sign-req$source-coopexec--aws--no-sign-requ$source-coopexec--aws--no-sign-reque$source-coopexec--aws--no-sign-reques$source-coopexec--aws--no-sign-request$source-coopexec--aws--no-sign-requests$source-coopexec--aws--no-sign-requests3$source-coopexec--aws--no-sign-requests3l$source-coopexec--aws--no-sign-requests3ls$source-coopexec--aws--no-sign-requests3lss$source-coopexec--aws--no-sign-requests3lss3$source-coopexec--aws--no-sign-requests3lss3:$source-coopexec--aws--no-sign-requests3lss3:/$source-coopexec--aws--no-sign-requests3lss3://partner-org$source-coopexec--aws--no-sign-requests3lss3://partner-org/ocean-buoys$source-coopexec--aws--no-sign-requests3lss3://partner-org/ocean-buoys/PREobservations/2026-08-0203:00:003072README.md2026-08-0203:00:007340032buoys.parquet$source-coopexec--aws--no-sign-requests3lss3://demo-org$source-coopexec--aws--no-sign-requests3lss3://demo-org/landsat-mirror$source-coopexec--aws--no-sign-requests3lss3://demo-org/landsat-mirror/;ecechoecho"exitecho"exit$?"Anerroroccurred(AccessDenied)whencallingtheListObjectsV2operation:accessdeniedexit254$source-coopexec--awss$source-coopexec--awss3l$source-coopexec--awss3lss$source-coopexec--awss3lss3$source-coopexec--awss3lss3://pa$source-coopexec--awss3lss3://par$source-coopexec--awss3lss3://part$source-coopexec--awss3lss3://partn$source-coopexec--awss3lss3://partne$source-coopexec--awss3lss3://partner$source-coopexec--awss3lss3://partner-$source-coopexec--awss3lss3://partner-o$source-coopexec--awss3lss3://partner-or$source-coopexec--awss3lss3://partner-org/o$source-coopexec--awss3lss3://partner-org/oc$source-coopexec--awss3lss3://partner-org/oce$source-coopexec--awss3lss3://partner-org/ocea$source-coopexec--awss3lss3://partner-org/ocean$source-coopexec--awss3lss3://partner-org/ocean-$source-coopexec--awss3lss3://partner-org/ocean-b$source-coopexec--awss3lss3://partner-org/ocean-bu$source-coopexec--awss3lss3://partner-org/ocean-buo$source-coopexec--awss3lss3://partner-org/ocean-buoy$source-coopexec--aws--no-sign-requests3lss3://pa$source-coopexec--aws--no-sign-requests3lss3://par$source-coopexec--aws--no-sign-requests3lss3://part$source-coopexec--aws--no-sign-requests3lss3://partn$source-coopexec--aws--no-sign-requests3lss3://partne$source-coopexec--aws--no-sign-requests3lss3://partner$source-coopexec--aws--no-sign-requests3lss3://partner-$source-coopexec--aws--no-sign-requests3lss3://partner-o$source-coopexec--aws--no-sign-requests3lss3://partner-or$source-coopexec--aws--no-sign-requests3lss3://partner-org/o$source-coopexec--aws--no-sign-requests3lss3://partner-org/oc$source-coopexec--aws--no-sign-requests3lss3://partner-org/oce$source-coopexec--aws--no-sign-requests3lss3://partner-org/ocea$source-coopexec--aws--no-sign-requests3lss3://partner-org/ocean$source-coopexec--aws--no-sign-requests3lss3://partner-org/ocean-$source-coopexec--aws--no-sign-requests3lss3://partner-org/ocean-b$source-coopexec--aws--no-sign-requests3lss3://partner-org/ocean-bu$source-coopexec--aws--no-sign-requests3lss3://partner-org/ocean-buo$source-coopexec--aws--no-sign-requests3lss3://partner-org/ocean-buoy$source-coopexec--aws--no-sign-requests3lss3://d$source-coopexec--aws--no-sign-requests3lss3://de$source-coopexec--aws--no-sign-requests3lss3://dem$source-coopexec--aws--no-sign-requests3lss3://demo$source-coopexec--aws--no-sign-requests3lss3://demo-$source-coopexec--aws--no-sign-requests3lss3://demo-o$source-coopexec--aws--no-sign-requests3lss3://demo-or$source-coopexec--aws--no-sign-requests3lss3://demo-org/l$source-coopexec--aws--no-sign-requests3lss3://demo-org/la$source-coopexec--aws--no-sign-requests3lss3://demo-org/lan$source-coopexec--aws--no-sign-requests3lss3://demo-org/land$source-coopexec--aws--no-sign-requests3lss3://demo-org/lands$source-coopexec--aws--no-sign-requests3lss3://demo-org/landsa$source-coopexec--aws--no-sign-requests3lss3://demo-org/landsat$source-coopexec--aws--no-sign-requests3lss3://demo-org/landsat-$source-coopexec--aws--no-sign-requests3lss3://demo-org/landsat-m$source-coopexec--aws--no-sign-requests3lss3://demo-org/landsat-mi$source-coopexec--aws--no-sign-requests3lss3://demo-org/landsat-mir$source-coopexec--aws--no-sign-requests3lss3://demo-org/landsat-mirr$source-coopexec--aws--no-sign-requests3lss3://demo-org/landsat-mirroeecho"echo"eecho"execho"exiecho"exit$echo"exit$? \ No newline at end of file diff --git a/docs/demos/exec.json b/docs/demos/exec.json new file mode 100644 index 0000000..e4f5567 --- /dev/null +++ b/docs/demos/exec.json @@ -0,0 +1,18 @@ +{ + "duration": 20.086, + "steps": [ + { + "time": 0.763, + "label": "$ source-coop exec -- aws s3 ls s3://demo-org/demo-data/" + }, + { + "time": 6.768, + "label": "$ source-coop exec -- s5cmd ls 's3://demo-org/demo-data/readings/*'" + }, + { + "time": 12.938, + "label": "$ source-coop exec -- env | grep -E '^AWS_(ACCESS_KEY_ID|ENDPOINT_URL_S3|REGION)='" + } + ], + "keys": [] +} \ No newline at end of file diff --git a/docs/demos/exec.svg b/docs/demos/exec.svg new file mode 100644 index 0000000..81f66b4 --- /dev/null +++ b/docs/demos/exec.svg @@ -0,0 +1 @@ +$$s$so$sou$sour$sourc$source$source-$source-c$source-co$source-coo$source-coop$source-coope$source-coopex$source-coopexe$source-coopexec$source-coopexec-$source-coopexec--$source-coopexec--aws$source-coopexec--awss3$source-coopexec--awss3ls$source-coopexec--awss3lss3:$source-coopexec--awss3lss3://demo-org$source-coopexec--awss3lss3://demo-org/demo-data$source-coopexec--awss3lss3://demo-org/demo-data/PREreadings/2026-09-1401:30:002048README.md2026-09-1401:30:0018874368stations.parquet$source-coopexec--s5cmd$source-coopexec--s5cmdls$source-coopexec--s5cmdls's3:$source-coopexec--s5cmdls's3://demo-org$source-coopexec--s5cmdls's3://demo-org/demo-data$source-coopexec--s5cmdls's3://demo-org/demo-data/readings$source-coopexec--s5cmdls's3://demo-org/demo-data/readings/*'2026/09/1408:30:00524288002026-08.parquet2026/09/1408:30:00503316482026-09.parquet$source-coopexec--env$source-coopexec--env|$source-coopexec--env|grep$source-coopexec--env|grep-E$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT_URL_S3|REGION)='AWS_ACCESS_KEY_ID=ASIADEMOEXAMPLE00001AWS_ENDPOINT_URL_S3=https://data.source.coopAWS_REGION=us-west-2$source-coopexec--a$source-coopexec--aw$source-coopexec--awss$source-coopexec--awss3l$source-coopexec--awss3lss$source-coopexec--awss3lss3$source-coopexec--awss3lss3:/$source-coopexec--awss3lss3://d$source-coopexec--awss3lss3://de$source-coopexec--awss3lss3://dem$source-coopexec--awss3lss3://demo$source-coopexec--awss3lss3://demo-$source-coopexec--awss3lss3://demo-o$source-coopexec--awss3lss3://demo-or$source-coopexec--awss3lss3://demo-org/d$source-coopexec--awss3lss3://demo-org/de$source-coopexec--awss3lss3://demo-org/dem$source-coopexec--awss3lss3://demo-org/demo$source-coopexec--awss3lss3://demo-org/demo-$source-coopexec--awss3lss3://demo-org/demo-d$source-coopexec--awss3lss3://demo-org/demo-da$source-coopexec--awss3lss3://demo-org/demo-dat$source-coopexec--s$source-coopexec--s5$source-coopexec--s5c$source-coopexec--s5cm$source-coopexec--s5cmdl$source-coopexec--s5cmdls'$source-coopexec--s5cmdls's$source-coopexec--s5cmdls's3$source-coopexec--s5cmdls's3:/$source-coopexec--s5cmdls's3://d$source-coopexec--s5cmdls's3://de$source-coopexec--s5cmdls's3://dem$source-coopexec--s5cmdls's3://demo$source-coopexec--s5cmdls's3://demo-$source-coopexec--s5cmdls's3://demo-o$source-coopexec--s5cmdls's3://demo-or$source-coopexec--s5cmdls's3://demo-org/d$source-coopexec--s5cmdls's3://demo-org/de$source-coopexec--s5cmdls's3://demo-org/dem$source-coopexec--s5cmdls's3://demo-org/demo$source-coopexec--s5cmdls's3://demo-org/demo-$source-coopexec--s5cmdls's3://demo-org/demo-d$source-coopexec--s5cmdls's3://demo-org/demo-da$source-coopexec--s5cmdls's3://demo-org/demo-dat$source-coopexec--s5cmdls's3://demo-org/demo-data/r$source-coopexec--s5cmdls's3://demo-org/demo-data/re$source-coopexec--s5cmdls's3://demo-org/demo-data/rea$source-coopexec--s5cmdls's3://demo-org/demo-data/read$source-coopexec--s5cmdls's3://demo-org/demo-data/readi$source-coopexec--s5cmdls's3://demo-org/demo-data/readin$source-coopexec--s5cmdls's3://demo-org/demo-data/reading$source-coopexec--s5cmdls's3://demo-org/demo-data/readings/*$source-coopexec--e$source-coopexec--en$source-coopexec--env|g$source-coopexec--env|gr$source-coopexec--env|gre$source-coopexec--env|grep-$source-coopexec--env|grep-E'$source-coopexec--env|grep-E'^$source-coopexec--env|grep-E'^A$source-coopexec--env|grep-E'^AW$source-coopexec--env|grep-E'^AWS$source-coopexec--env|grep-E'^AWS_$source-coopexec--env|grep-E'^AWS_($source-coopexec--env|grep-E'^AWS_(A$source-coopexec--env|grep-E'^AWS_(AC$source-coopexec--env|grep-E'^AWS_(ACC$source-coopexec--env|grep-E'^AWS_(ACCE$source-coopexec--env|grep-E'^AWS_(ACCES$source-coopexec--env|grep-E'^AWS_(ACCESS$source-coopexec--env|grep-E'^AWS_(ACCESS_$source-coopexec--env|grep-E'^AWS_(ACCESS_K$source-coopexec--env|grep-E'^AWS_(ACCESS_KE$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_I$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|E$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|EN$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|END$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDP$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPO$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOI$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOIN$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT_$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT_U$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT_UR$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT_URL$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT_URL_$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT_URL_S$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT_URL_S3$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT_URL_S3|$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT_URL_S3|R$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT_URL_S3|RE$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT_URL_S3|REG$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT_URL_S3|REGI$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT_URL_S3|REGIO$source-coopexec--env|grep-E'^AWS_(ACCESS_KEY_ID|ENDPOINT_URL_S3|REGION= \ No newline at end of file diff --git a/docs/recorder/fixtures.json b/docs/recorder/fixtures.json index 0f5892e..9fef806 100644 --- a/docs/recorder/fixtures.json +++ b/docs/recorder/fixtures.json @@ -464,6 +464,24 @@ "key": "demo-data/readings/2026-09.parquet", "size": 50331648, "last_modified": "2026-09-14T08:30:00Z" + }, + { + "bucket": "partner-org", + "key": "ocean-buoys/README.md", + "size": 3072, + "last_modified": "2026-08-02T10:00:00Z" + }, + { + "bucket": "partner-org", + "key": "ocean-buoys/buoys.parquet", + "size": 7340032, + "last_modified": "2026-08-02T10:00:00Z" + }, + { + "bucket": "partner-org", + "key": "ocean-buoys/observations/2026.parquet", + "size": 94371840, + "last_modified": "2026-10-01T10:00:00Z" } ] } diff --git a/docs/recorder/guide.py b/docs/recorder/guide.py index d6a6c46..b74472a 100644 --- a/docs/recorder/guide.py +++ b/docs/recorder/guide.py @@ -82,6 +82,19 @@ def aws(s): s.run("source-coop creds --format env") +def exec_signed_in(s): + signed_in(s) + s.run(f"source-coop exec -- aws s3 ls s3://{ORG}/demo-data/"); s.prompt(); s.pause(1.5) + s.run(f"source-coop exec -- s5cmd ls 's3://{ORG}/demo-data/readings/*'"); s.prompt(); s.pause(1.5) + s.run("source-coop exec -- env | grep -E '^AWS_(ACCESS_KEY_ID|ENDPOINT_URL_S3|REGION)='") + + +def exec_signed_out(s): + s.run("source-coop exec -- aws s3 ls s3://partner-org/ocean-buoys/"); s.prompt(); s.pause(3) + s.run("source-coop exec -- aws --no-sign-request s3 ls s3://partner-org/ocean-buoys/"); s.prompt(); s.pause(2) + s.run(f"source-coop exec -- aws --no-sign-request s3 ls s3://{ORG}/landsat-mirror/; echo \"exit $?\"") + + def prompts(s): s.run("source-coop product create") s.see("Account"); s.key(DOWN); s.enter() @@ -397,6 +410,8 @@ def service_account_usage(s): "auth-token": ("`auth token`: the session's access token, for `curl`", auth_token), "auth-logout": ("`auth logout --json`, then `auth status` and `creds` signed out", auth_logout), "aws": ("`aws s3 ls` through `credential_process`, then `creds --format env`", aws), + "exec": ("`exec`: the AWS CLI and s5cmd with your credentials and the proxy, no profile", exec_signed_in), + "exec-signed-out": ("`exec` signed out: unsigned, a public product listed, a restricted one refused", exec_signed_out), "prompts": ("Prompts: a refused ID asked again, a draft kept after a failure, then Esc", prompts), "completion": ("Shell completion: TAB lists products and service accounts from the API", completion), "account-list": ("`account list`, `view`, then `list --json`", account_list), diff --git a/docs/recorder/stub.py b/docs/recorder/stub.py index 02f1918..8cbd2f8 100644 --- a/docs/recorder/stub.py +++ b/docs/recorder/stub.py @@ -6,7 +6,10 @@ it checks nothing, so it is no substitute for the API in tests. Outside /api/v1 it is the data proxy, for `aws s3 ls`: a ListObjectsV2 of -fixtures.json's `objects`, signatures unchecked. +fixtures.json's `objects`, signatures unchecked. An unsigned listing of a +product that isn't public is refused, as the proxy refuses it. /oauth2/token +refuses to renew a session, as the sign-in server does once a refresh token +has expired. python3 -I docs/recorder/stub.py PORT [FIXTURES] """ @@ -260,6 +263,18 @@ def list_objects(bucket, q): + tag("IsTruncated", "false") + body + "") +def readable_unsigned(bucket, q): + """Whether the proxy lists `bucket` and `prefix` for a request without a + signature: only inside a public product.""" + product = q.get("prefix", "").split("/")[0] + return any(p["account_id"] == bucket and p["product_id"] == product and p["visibility"] == "public" + for p in products) + + +ACCESS_DENIED = ('' + "AccessDeniedaccess denied") + + class Handler(BaseHTTPRequestHandler): def log_message(self, *args): pass @@ -267,9 +282,24 @@ def log_message(self, *args): def answer(self, method): url = urlparse(self.path) q = {k: v[0] for k, v in parse_qs(url.query).items()} + if url.path == "/oauth2/token": + # The sign-in server, refusing to renew a session: a refresh + # token that expired or was revoked. + raw = json.dumps({"error": "invalid_grant", + "error_description": "The refresh token has expired."}).encode() + self.send_response(400) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + return if not url.path.startswith("/api/v1"): - raw = list_objects(url.path.strip("/").split("/")[0], q).encode() - self.send_response(200) + bucket = url.path.strip("/").split("/")[0] + if self.headers.get("Authorization") or readable_unsigned(bucket, q): + status, raw = 200, list_objects(bucket, q).encode() + else: + status, raw = 403, ACCESS_DENIED.encode() + self.send_response(status) self.send_header("Content-Type", "application/xml") self.send_header("Content-Length", str(len(raw))) self.end_headers() diff --git a/src/exec.rs b/src/exec.rs index 95baaef..9c6eaeb 100644 --- a/src/exec.rs +++ b/src/exec.rs @@ -1,21 +1,63 @@ -//! `source-coop exec`: run any S3 tool (the AWS CLI, s5cmd, rclone, an SDK -//! script) as the signed-in user, with credentials and the data proxy's -//! endpoint in its environment, as `aws-vault exec` does for AWS. +//! `source-coop exec`: run any S3 tool (the AWS CLI, s5cmd, GDAL, an SDK +//! script) with the data proxy as its endpoint, as `aws-vault exec` does for +//! AWS: as the signed-in user, with credentials in its environment, or, when +//! nobody is signed in, unsigned, which reads public products. use crate::sts::Credentials; use std::process::Command; -/// The environment `command` runs with: the credentials, where they expire, -/// and the proxy as the S3 endpoint under the names the common tools read. -/// A region is only given when none is set, as the proxy ignores it but -/// signing needs one. -pub fn vars(creds: &Credentials, proxy_url: &str, region_set: bool) -> Vec<(&'static str, String)> { +/// The region a command is given when it has none: the proxy ignores it, but +/// signing needs one. The same as `service-account usage` gives. +const REGION: &str = "us-west-2"; + +/// What `command` runs with: variables to set, and variables to remove. +#[derive(Debug, Default, PartialEq)] +pub struct Env { + pub set: Vec<(&'static str, String)>, + pub remove: Vec<&'static str>, +} + +/// Removed whether signed or not. A profile could hold other credentials or +/// an endpoint; the env vars already win, but naming one invites confusion +/// over which applied. And tools told to ignore configured endpoints would +/// send requests to AWS itself rather than the proxy. +const ALWAYS_REMOVED: &[&str] = &[ + "AWS_PROFILE", + "AWS_DEFAULT_PROFILE", + "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", +]; + +/// Removed when running unsigned: every other way an AWS tool finds keys. +/// Keys the proxy doesn't know (anyone's real AWS keys) are refused even for +/// public data, and would be sent, if not the secret, to the proxy. +const OTHER_CREDENTIALS: &[&str] = &[ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_SECURITY_TOKEN", + "AWS_CREDENTIAL_EXPIRATION", + "AWS_ROLE_ARN", + "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_ROLE_SESSION_NAME", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", + "AWS_CONTAINER_AUTHORIZATION_TOKEN", + "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", +]; + +/// A file that reads as empty, for the AWS config and credentials files. +#[cfg(unix)] +const EMPTY_FILE: &str = "/dev/null"; +#[cfg(not(unix))] +const EMPTY_FILE: &str = "NUL"; + +/// The environment `command` runs with: the proxy as the S3 endpoint under +/// the names the common tools read, and either `creds` and where they expire +/// or, without them, each tool's switch to send requests unsigned. A region +/// is only given when none is set. +pub fn env(creds: Option<&Credentials>, proxy_url: &str, region_set: bool) -> Env { let endpoint = proxy_url.trim_end_matches('/').to_string(); - let mut vars = vec![ - ("AWS_ACCESS_KEY_ID", creds.access_key_id.clone()), - ("AWS_SECRET_ACCESS_KEY", creds.secret_access_key.clone()), - ("AWS_SESSION_TOKEN", creds.session_token.clone()), - ("AWS_CREDENTIAL_EXPIRATION", creds.expiration.clone()), + let mut set = vec![ // The AWS CLI (v2.13+) and SDKs, for S3 alone: other services (STS, // KMS) a script calls keep their own endpoints. ("AWS_ENDPOINT_URL_S3", endpoint.clone()), @@ -29,7 +71,7 @@ pub fn vars(creds: &Credentials, proxy_url: &str, region_set: bool) -> Vec<(&'st let http = url.scheme() == "http"; if http { // `object_store` refuses plain HTTP (a local proxy) without it. - vars.push(("AWS_ALLOW_HTTP", "true".into())); + set.push(("AWS_ALLOW_HTTP", "true".into())); } // GDAL's /vsis3/ (QGIS, rasterio, geopandas), which reads no AWS // endpoint setting: a host without the scheme for GDAL before 3.11, @@ -39,36 +81,79 @@ pub fn vars(creds: &Credentials, proxy_url: &str, region_set: bool) -> Vec<(&'st Some(port) => format!("{host}:{port}"), None => host.to_string(), }; - vars.push(("AWS_S3_ENDPOINT", host)); - vars.push(("AWS_HTTPS", if http { "NO" } else { "YES" }.into())); - vars.push(("AWS_VIRTUAL_HOSTING", "FALSE".into())); + set.push(("AWS_S3_ENDPOINT", host)); + set.push(("AWS_HTTPS", if http { "NO" } else { "YES" }.into())); + set.push(("AWS_VIRTUAL_HOSTING", "FALSE".into())); } } if !region_set { - vars.push(("AWS_REGION", "us-east-1".into())); - vars.push(("AWS_DEFAULT_REGION", "us-east-1".into())); + set.push(("AWS_REGION", REGION.into())); + set.push(("AWS_DEFAULT_REGION", REGION.into())); + } + let mut remove = ALWAYS_REMOVED.to_vec(); + match creds { + Some(creds) => set.extend([ + ("AWS_ACCESS_KEY_ID", creds.access_key_id.clone()), + ("AWS_SECRET_ACCESS_KEY", creds.secret_access_key.clone()), + ("AWS_SESSION_TOKEN", creds.session_token.clone()), + ("AWS_CREDENTIAL_EXPIRATION", creds.expiration.clone()), + ]), + None => { + remove.extend(OTHER_CREDENTIALS); + set.extend([ + // GDAL. + ("AWS_NO_SIGN_REQUEST", "YES".into()), + // `object_store`. + ("AWS_SKIP_SIGNATURE", "true".into()), + // Keys in ~/.aws, or from an EC2 instance's role, would sign. + ("AWS_SHARED_CREDENTIALS_FILE", EMPTY_FILE.into()), + ("AWS_CONFIG_FILE", EMPTY_FILE.into()), + ("AWS_EC2_METADATA_DISABLED", "true".into()), + ]); + } } - vars + Env { set, remove } } -/// Run `command` with `vars` added to the environment, and leave with its -/// exit code. On Unix this process becomes the command, so signals (Ctrl+C) -/// and the exit status reach the shell as if it had been run directly. -pub fn run(command: &[String], vars: Vec<(&'static str, String)>) -> Result<(), String> { +/// For a command run unsigned, how to tell it so when no variable can: the +/// AWS CLI and s5cmd sign every request unless given `--no-sign-request`, +/// and fail without credentials. None when the command has the flag, or +/// isn't one of those. +pub fn unsigned_hint(command: &[String]) -> Option { + let (program, args) = command.split_first()?; + let name = std::path::Path::new(program).file_stem()?.to_str()?; + if !matches!(name, "aws" | "s5cmd") || args.iter().any(|a| a == "--no-sign-request") { + return None; + } + let args: Vec<_> = args.iter().map(|a| quoted(a)).collect(); + Some(format!( + "{name} needs --no-sign-request to run unsigned: \ + source-coop exec -- {program} --no-sign-request {}", + args.join(" ") + )) +} + +/// `arg` as a shell would need it typed: in single quotes when it holds +/// anything but the characters that need none. +fn quoted(arg: &str) -> String { + let plain = |c: char| c.is_ascii_alphanumeric() || "-_./:=@%+,".contains(c); + if !arg.is_empty() && arg.chars().all(plain) { + return arg.to_string(); + } + format!("'{}'", arg.replace('\'', r"'\''")) +} + +/// Run `command` in `env`, and leave with its exit code. On Unix this +/// process becomes the command, so signals (Ctrl+C) and the exit status reach +/// the shell as if it had been run directly. +pub fn run(command: &[String], env: Env) -> Result<(), String> { let (program, args) = command.split_first().ok_or("Name a command to run.")?; let mut cmd = Command::new(program); - cmd.args(args).envs(vars); - // A profile could hold other credentials or an endpoint; the env vars - // already win, but naming one invites confusion over which applied. And - // tools told to ignore configured endpoints would send the credentials - // to AWS itself rather than the proxy. - for name in [ - "AWS_PROFILE", - "AWS_DEFAULT_PROFILE", - "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", - ] { + cmd.args(args); + for name in env.remove { cmd.env_remove(name); } + cmd.envs(env.set); let failed = |e: std::io::Error| format!("Couldn't run {program}: {e}"); #[cfg(unix)] @@ -97,56 +182,135 @@ mod tests { } } + fn get<'a>(env: &'a Env, name: &str) -> Option<&'a str> { + env.set + .iter() + .find(|(n, _)| *n == name) + .map(|(_, v)| v.as_str()) + } + #[test] fn the_command_gets_credentials_and_the_proxy_as_its_endpoint() { - let vars = vars(&creds(), "https://data.staging.source.coop/", false); - let get = |k: &str| { - vars.iter() - .find(|(name, _)| *name == k) - .map(|(_, v)| v.as_str()) - }; - assert_eq!(get("AWS_ACCESS_KEY_ID"), Some("AKID")); - assert_eq!(get("AWS_SESSION_TOKEN"), Some("TOKEN")); + let env = env(Some(&creds()), "https://data.staging.source.coop/", false); + assert_eq!(get(&env, "AWS_ACCESS_KEY_ID"), Some("AKID")); + assert_eq!(get(&env, "AWS_SECRET_ACCESS_KEY"), Some("SECRET")); + assert_eq!(get(&env, "AWS_SESSION_TOKEN"), Some("TOKEN")); assert_eq!( - get("AWS_CREDENTIAL_EXPIRATION"), + get(&env, "AWS_CREDENTIAL_EXPIRATION"), Some("2026-10-14T13:00:00Z") ); - for name in ["AWS_ENDPOINT_URL_S3", "S3_ENDPOINT_URL"] { + for name in ["AWS_ENDPOINT_URL_S3", "S3_ENDPOINT_URL", "AWS_ENDPOINT"] { assert_eq!( - get(name), + get(&env, name), Some("https://data.staging.source.coop"), "{name}" ); } - assert_eq!(get("AWS_ENDPOINT_URL"), None, "only S3 goes to the proxy"); assert_eq!( - get("AWS_ENDPOINT"), - Some("https://data.staging.source.coop") + get(&env, "AWS_ENDPOINT_URL"), + None, + "only S3 goes to the proxy" + ); + assert_eq!( + get(&env, "AWS_S3_ENDPOINT"), + Some("data.staging.source.coop") + ); + assert_eq!(get(&env, "AWS_HTTPS"), Some("YES")); + assert_eq!(get(&env, "AWS_VIRTUAL_HOSTING"), Some("FALSE")); + assert_eq!(get(&env, "AWS_ALLOW_HTTP"), None); + assert_eq!(get(&env, "AWS_REGION"), Some("us-west-2")); + assert_eq!(get(&env, "AWS_DEFAULT_REGION"), Some("us-west-2")); + // Signed: nothing says to skip signing, and only profiles go. + for name in [ + "AWS_NO_SIGN_REQUEST", + "AWS_SKIP_SIGNATURE", + "AWS_CONFIG_FILE", + ] { + assert_eq!(get(&env, name), None, "{name}"); + } + assert_eq!(env.remove, ALWAYS_REMOVED); + } + + #[test] + fn without_credentials_the_command_runs_unsigned_and_ignores_other_keys() { + let env = env(None, "https://data.source.coop", false); + assert_eq!( + get(&env, "AWS_ENDPOINT_URL_S3"), + Some("https://data.source.coop") ); - assert_eq!(get("AWS_S3_ENDPOINT"), Some("data.staging.source.coop")); - assert_eq!(get("AWS_HTTPS"), Some("YES")); - assert_eq!(get("AWS_VIRTUAL_HOSTING"), Some("FALSE")); - assert_eq!(get("AWS_ALLOW_HTTP"), None); - assert_eq!(get("AWS_REGION"), Some("us-east-1")); + assert_eq!(get(&env, "AWS_S3_ENDPOINT"), Some("data.source.coop")); + assert_eq!(get(&env, "AWS_REGION"), Some("us-west-2")); + assert_eq!(get(&env, "AWS_NO_SIGN_REQUEST"), Some("YES")); + assert_eq!(get(&env, "AWS_SKIP_SIGNATURE"), Some("true")); + assert_eq!(get(&env, "AWS_SHARED_CREDENTIALS_FILE"), Some(EMPTY_FILE)); + assert_eq!(get(&env, "AWS_CONFIG_FILE"), Some(EMPTY_FILE)); + assert_eq!(get(&env, "AWS_EC2_METADATA_DISABLED"), Some("true")); + for name in [ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + ] { + assert_eq!(get(&env, name), None, "{name} isn't set"); + assert!(env.remove.contains(&name), "{name} is removed"); + } + for name in ["AWS_PROFILE", "AWS_ROLE_ARN", "AWS_WEB_IDENTITY_TOKEN_FILE"] { + assert!(env.remove.contains(&name), "{name} is removed"); + } + // Nothing is both set and removed. + assert!(env.set.iter().all(|(n, _)| !env.remove.contains(n))); } #[test] fn a_local_proxy_over_http_keeps_its_port() { - let vars = vars(&creds(), "http://localhost:9000", false); - let get = |k: &str| vars.iter().find(|(n, _)| *n == k).map(|(_, v)| v.as_str()); - assert_eq!(get("AWS_S3_ENDPOINT"), Some("localhost:9000")); - assert_eq!(get("AWS_HTTPS"), Some("NO")); - assert_eq!(get("AWS_ALLOW_HTTP"), Some("true")); + let env = env(Some(&creds()), "http://localhost:9000", false); + assert_eq!(get(&env, "AWS_S3_ENDPOINT"), Some("localhost:9000")); + assert_eq!(get(&env, "AWS_HTTPS"), Some("NO")); + assert_eq!(get(&env, "AWS_ALLOW_HTTP"), Some("true")); } #[test] fn a_region_already_set_is_kept() { - let vars = vars(&creds(), "https://data.source.coop", true); - assert!(!vars.iter().any(|(name, _)| name.contains("REGION"))); + for creds in [Some(creds()), None] { + let env = env(creds.as_ref(), "https://data.source.coop", true); + assert!(!env.set.iter().any(|(name, _)| name.contains("REGION"))); + } + } + + #[test] + fn the_aws_cli_and_s5cmd_are_told_to_add_no_sign_request() { + let cmd = |s: &str| s.split(' ').map(String::from).collect::>(); + assert_eq!( + unsigned_hint(&cmd("aws s3 ls s3://acct/p/")).as_deref(), + Some( + "aws needs --no-sign-request to run unsigned: \ + source-coop exec -- aws --no-sign-request s3 ls s3://acct/p/" + ) + ); + let s5cmd = vec![ + "/usr/local/bin/s5cmd".to_string(), + "cp".into(), + "s3://acct/p/*.parquet".into(), + "it's here".into(), + ]; + assert_eq!( + unsigned_hint(&s5cmd).as_deref(), + Some( + "s5cmd needs --no-sign-request to run unsigned: source-coop exec -- \ + /usr/local/bin/s5cmd --no-sign-request cp 's3://acct/p/*.parquet' 'it'\\''s here'" + ) + ); + assert_eq!( + unsigned_hint(&cmd("aws --no-sign-request s3 ls s3://acct/p/")), + None + ); + assert_eq!(unsigned_hint(&cmd("gdalinfo /vsis3/acct/p/a.tif")), None); } #[test] fn a_missing_command_is_an_error() { - assert_eq!(run(&[], vec![]), Err("Name a command to run.".to_string())); + assert_eq!( + run(&[], Env::default()), + Err("Name a command to run.".to_string()) + ); } } diff --git a/src/main.rs b/src/main.rs index 5cdea2a..e7b54a7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -73,6 +73,7 @@ For scripts and agents: SOURCE_TOKEN A bearer token for the API, used instead of the login session. api Any API request; prints the API's JSON as is, without --json. Paths and fields are in the OpenAPI spec at {api}/api/openapi. + exec Reads and writes data with S3 tools: `exec -- aws s3 ls s3://ACCOUNT/PRODUCT/`. Exit codes: 0 success 1 other failure 2 bad usage 3 input rejected (fix field_errors) @@ -128,12 +129,37 @@ enum Commands { /// commands don't cover #[command(after_help = style::help(&API_HELP.replace("{api}", defaults::API_URL)))] Api(ApiCommandArgs), - /// Run a command (the AWS CLI, s5cmd, an SDK script) with credentials and - /// the data proxy as its S3 endpoint in its environment - /// (e.g. `source-coop exec -- aws s3 ls s3://my-org/my-product/`) + /// Run a command (the AWS CLI, s5cmd, GDAL, a script) with the data proxy + /// as its S3 endpoint and your credentials, or unsigned when signed out + #[command(after_help = style::help(EXEC_HELP))] Exec(ExecArgs), } +/// `source-coop exec --help` after the flags: examples, then what the +/// command gets and what happens signed out. +const EXEC_HELP: &str = "\ +Examples: + source-coop exec -- aws s3 ls s3://ACCOUNT/PRODUCT/ + source-coop exec -- aws s3 cp s3://ACCOUNT/PRODUCT/data.parquet . + source-coop exec -- gdalinfo /vsis3/ACCOUNT/PRODUCT/image.tif + source-coop exec -- python script.py + +The command gets: + AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, AWS_CREDENTIAL_EXPIRATION + The proxy as its S3 endpoint: AWS_ENDPOINT_URL_S3 (AWS CLI, SDKs), S3_ENDPOINT_URL + (s5cmd), AWS_ENDPOINT (object_store), AWS_S3_ENDPOINT (GDAL) + AWS_REGION=us-west-2, if no region is set +Credentials are fetched once, with at least 15 minutes left. The command's exit code is +passed on. + +Signed out: + The command runs unsigned and can read only public data. GDAL and object_store are told + so (AWS_NO_SIGN_REQUEST, AWS_SKIP_SIGNATURE); the AWS CLI and s5cmd need + --no-sign-request: + source-coop exec -- aws --no-sign-request s3 ls s3://ACCOUNT/PRODUCT/ + Other AWS keys in the environment or ~/.aws are hidden from it. A session that can't be + renewed is an error, not an unsigned run."; + #[derive(Parser)] struct ExecArgs { #[command(flatten)] @@ -1017,32 +1043,66 @@ async fn run_login(args: LoginArgs, verbose: bool) -> Result<(), String> { Ok(()) } -/// Run the command with the role's credentials. They're fetched once, so -/// cached ones are only reused with `EXEC_MIN_LIFETIME` left; a command +/// Run the command with the role's credentials, or unsigned when nobody is +/// signed in, so it can still read public products. They're fetched once, +/// so cached ones are only reused with `EXEC_MIN_LIFETIME` left; a command /// running past their expiry (an hour unless `login --duration` said /// otherwise) needs running again. async fn run_exec(args: ExecArgs, verbose: bool) -> Result<(), String> { - let entry = current_creds( + let creds = current_creds( &args.role.role_arn, &args.env.proxy_url, EXEC_MIN_LIFETIME, verbose, ) - .await?; + .await; + let (env, notes) = exec_env(&args.command, &args.env.proxy_url, signed(creds)?); + for note in notes { + eprintln!("{note}"); + } + if verbose { + eprintln!( + "[verbose] Running {:?} against {}", + args.command, args.env.proxy_url + ); + } + exec::run(&args.command, env) +} + +/// The credentials `exec` signs with, or None to run unsigned: only when +/// nobody is signed in. A session that can't be renewed is an error instead, +/// since whoever signed in means to act as themselves, and an unsigned run +/// would turn that into a puzzling refusal of their own data. +fn signed(creds: Result) -> Result, String> { + match creds { + Ok(entry) => Ok(Some(entry.creds)), + Err(e) if e == NOT_SIGNED_IN => Ok(None), + Err(e) => Err(e), + } +} + +/// The environment `exec` runs `command` in, and the notes to print first. +fn exec_env( + command: &[String], + proxy_url: &str, + creds: Option, +) -> (exec::Env, Vec) { // Set but empty is unset, as signing would fail on an empty region. let region_set = ["AWS_REGION", "AWS_DEFAULT_REGION"] .iter() .any(|v| std::env::var_os(v).is_some_and(|r| !r.is_empty())); - let vars = exec::vars(&entry.creds, &args.env.proxy_url, region_set); - if verbose { - eprintln!( - "[verbose] Running {:?} against {} with credentials expiring {}", - args.command, args.env.proxy_url, entry.creds.expiration - ); + let env = exec::env(creds.as_ref(), proxy_url, region_set); + let mut notes = vec![]; + if creds.is_none() { + notes.push(UNSIGNED_NOTE.to_string()); + notes.extend(exec::unsigned_hint(command)); } - exec::run(&args.command, vars) + (env, notes) } +/// What `exec` says before running a command unsigned. +const UNSIGNED_NOTE: &str = "Not signed in, so running unsigned: only public data can be read."; + /// How long cached credentials must still last for `exec` to hand them to a /// command, which can't ask for new ones as `credential_process` can. const EXEC_MIN_LIFETIME: chrono::Duration = chrono::Duration::minutes(15); @@ -1094,7 +1154,7 @@ async fn current_creds( stale => mint(role_arn, proxy_url, env, stale, verbose) .await .map_err(|e| match e.as_str() { - NOT_LOGGED_IN => "Not signed in. Run 'source-coop auth login'.".to_string(), + NOT_LOGGED_IN => NOT_SIGNED_IN.to_string(), _ => format!("Couldn't get new credentials ({e}). Run 'source-coop auth login'."), }), } @@ -1103,6 +1163,9 @@ async fn current_creds( /// `mint`'s error when there's no login session to mint with. const NOT_LOGGED_IN: &str = "not logged in"; +/// `current_creds`'s error when there's no login session. +const NOT_SIGNED_IN: &str = "Not signed in. Run 'source-coop auth login'."; + /// New credentials for the role, cached before they're returned. An entry /// from before the session was shared refreshes with its own token; anything /// else exchanges the session's ID token, refreshing the session if need be. @@ -1753,6 +1816,55 @@ mod tests { assert!(broken.is_empty(), "{}", broken.join("\n\n")); } + #[test] + fn exec_runs_unsigned_only_when_nobody_is_signed_in() { + let creds = sts::Credentials { + access_key_id: "AKID".into(), + secret_access_key: "SECRET".into(), + session_token: "TOKEN".into(), + expiration: "2026-10-14T13:00:00Z".into(), + }; + let entry = cache::CacheEntry { + creds: creds.clone(), + sts: None, + refresh: None, + }; + assert_eq!(signed(Ok(entry)), Ok(Some(creds))); + assert_eq!(signed(Err(NOT_SIGNED_IN.into())), Ok(None)); + // Signed in, but the session can't be renewed: an error, not unsigned. + let refused = "Couldn't get new credentials (invalid_grant). Run 'source-coop auth login'."; + assert_eq!(signed(Err(refused.into())), Err(refused.into())); + } + + #[test] + fn exec_says_when_it_runs_unsigned() { + let cmd = |s: &str| s.split(' ').map(String::from).collect::>(); + let proxy = "https://data.source.coop"; + + let (env, notes) = exec_env(&cmd("aws s3 ls s3://acct/p/"), proxy, None); + assert_eq!( + notes, + [ + UNSIGNED_NOTE.to_string(), + exec::unsigned_hint(&cmd("aws s3 ls s3://acct/p/")).unwrap() + ] + ); + assert!(env.remove.contains(&"AWS_ACCESS_KEY_ID")); + + let (_, notes) = exec_env(&cmd("gdalinfo /vsis3/acct/p/a.tif"), proxy, None); + assert_eq!(notes, [UNSIGNED_NOTE]); + + let creds = sts::Credentials { + access_key_id: "AKID".into(), + secret_access_key: "SECRET".into(), + session_token: "TOKEN".into(), + expiration: "2026-10-14T13:00:00Z".into(), + }; + let (env, notes) = exec_env(&cmd("aws s3 ls s3://acct/p/"), proxy, Some(creds)); + assert!(notes.is_empty(), "signed in, nothing to say: {notes:?}"); + assert!(env.set.contains(&("AWS_ACCESS_KEY_ID", "AKID".into()))); + } + #[test] fn every_command_is_in_the_readme() { let readme = include_str!("../README.md"); diff --git a/src/sts.rs b/src/sts.rs index 90b470d..b7ed931 100644 --- a/src/sts.rs +++ b/src/sts.rs @@ -1,7 +1,7 @@ use quick_xml::de::from_str as xml_from_str; use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Credentials { pub access_key_id: String, pub secret_access_key: String, From c62543bd50254f06f1ae5d5cf1b7a6f3e1d49aed Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Mon, 9 Nov 2026 10:58:45 -0700 Subject: [PATCH 4/4] fix: exec signs with its own credentials whatever the environment holds From review: - Signed in, exec removes AWS_NO_SIGN_REQUEST and AWS_SKIP_SIGNATURE, so GDAL and object_store sign even if they were exported for public buckets. - exec always removes a general AWS_ENDPOINT_URL, which object_store reads as its S3 endpoint and could win over the proxy's AWS_ENDPOINT. - "Nobody signed in" is an Option from current_creds, not a match on an error's text. - The --no-sign-request hint quotes the program as well as its arguments. - creds and the API commands share one "Not signed in" message. README: the signed-out creds message and its exit code in Troubleshooting, and the auth-logout demo re-recorded to show them. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- docs/demos/auth-logout.json | 6 +-- docs/demos/auth-logout.svg | 2 +- src/api.rs | 7 ++-- src/exec.rs | 52 ++++++++++++++++++------- src/main.rs | 75 +++++++++++-------------------------- 6 files changed, 71 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 3ea15d4..1b6cb6a 100644 --- a/README.md +++ b/README.md @@ -310,7 +310,7 @@ The command gets: - **The proxy as its S3 endpoint**, under each tool's name for it: `AWS_ENDPOINT_URL_S3` (the AWS CLI v2.13+ and SDKs), `S3_ENDPOINT_URL` (s5cmd), `AWS_ENDPOINT` (`object_store`, with `AWS_ALLOW_HTTP` for a plain-HTTP proxy), and `AWS_S3_ENDPOINT`, `AWS_HTTPS` and `AWS_VIRTUAL_HOSTING=FALSE` (GDAL's `/vsis3/`). Only S3 goes to the proxy: calls a script makes to other AWS services keep their own endpoints. - **A region**, `AWS_REGION` and `AWS_DEFAULT_REGION` set to `us-west-2`, only if neither is set already, as signing needs one. -`AWS_PROFILE`, `AWS_DEFAULT_PROFILE` and `AWS_IGNORE_CONFIGURED_ENDPOINT_URLS` are removed, so a profile can't lend other credentials or send these anywhere but the proxy. `--role-arn` and `--proxy-url` (or `SOURCE_ROLE_ARN` and `SOURCE_PROXY_URL`) select the role and environment, as for `creds`. +`AWS_PROFILE`, `AWS_DEFAULT_PROFILE`, `AWS_IGNORE_CONFIGURED_ENDPOINT_URLS` and `AWS_ENDPOINT_URL` are removed, so a profile can't lend other credentials and no other endpoint can take S3 requests away from the proxy. (`object_store` reads a general `AWS_ENDPOINT_URL` as its S3 endpoint.) Signed in, `AWS_NO_SIGN_REQUEST` and `AWS_SKIP_SIGNATURE` are removed too, so GDAL and `object_store` sign even if you exported them for public buckets. `--role-arn` and `--proxy-url` (or `SOURCE_ROLE_ARN` and `SOURCE_PROXY_URL`) select the role and environment, as for `creds`. The credentials are fetched once, when the command starts. Cached ones are used if they have at least 15 minutes left. Otherwise `exec` gets new ones. A command still running when they expire (an hour after they're fetched, unless you signed in with `--duration`) must be run again. For long jobs, use the [`credential_process` profile](#reading-data-with-aws-tools), which renews them. @@ -1252,6 +1252,8 @@ The session is in `source-coop/session.json`. Other environments use the same la **`Your login has expired. Run 'source-coop auth login'.`** The session's refresh token expired or was revoked. Sign in again. `source-coop auth status` estimates when it expires. +**`Not signed in: run 'source-coop auth login'.`** Nobody is signed in to this environment (`--proxy-url`). `creds` exits with code 4; AWS tools that run it through `credential_process` show the message as their own error. Sign in, or read public data without credentials (see [Access Data](https://docs.source.coop/data-proxy)). `exec` doesn't fail here: it runs the command unsigned. + **`Couldn't get new credentials (…). Run 'source-coop auth login'.`** `creds` or `exec` could not renew a role's credentials. AWS tools show this message as their own error. The reason is in the brackets. Usually the session has expired, so sign in again. **`Unable to locate credentials` or `NoCredentialProviders` under `exec`.** You are not signed in, so `exec` runs the command unsigned, and the AWS CLI and s5cmd then need `--no-sign-request`. Add it to read public data, or run `source-coop auth login`. See [Run a tool with `exec`](#run-a-tool-with-exec). diff --git a/docs/demos/auth-logout.json b/docs/demos/auth-logout.json index 0be4456..07a55f5 100644 --- a/docs/demos/auth-logout.json +++ b/docs/demos/auth-logout.json @@ -1,16 +1,16 @@ { - "duration": 13.88, + "duration": 14.59, "steps": [ { "time": 0.757, "label": "$ source-coop auth logout --json | jq -c" }, { - "time": 5.372, + "time": 6.055, "label": "$ source-coop auth status" }, { - "time": 9.241, + "time": 9.938, "label": "$ source-coop creds; echo \"exit $?\"" } ], diff --git a/docs/demos/auth-logout.svg b/docs/demos/auth-logout.svg index 36913b2..edd4bfc 100644 --- a/docs/demos/auth-logout.svg +++ b/docs/demos/auth-logout.svg @@ -1 +1 @@ -$$s$so$sou$sour$sourc$source$source-$source-c$source-co$source-coo$source-coop$source-coopa$source-coopau$source-coopaut$source-coopauth$source-coopauthlogout$source-coopauthlogout--json$source-coopauthlogout--json|$source-coopauthlogout--json|jq$source-coopauthlogout--json|jq-c{"proxy_url":"https://data.source.coop","removed":{"credentials":true,"session":true},"role_arn":"_default"}$source-coopauthstatusEnvironmenthttps://data.source.coopLoginnotloggedinCredentials_default:nonecached$source-coopcreds;$source-coopcreds;ec$source-coopcreds;echo$source-coopcreds;echo"exit$source-coopcreds;echo"exit$?"Error:Couldn'tgetnewcredentials(notloggedin).Run'source-coopauthlogin'.exit1$source-coopauthl$source-coopauthlo$source-coopauthlog$source-coopauthlogo$source-coopauthlogou$source-coopauthlogout-$source-coopauthlogout--$source-coopauthlogout--j$source-coopauthlogout--js$source-coopauthlogout--jso$source-coopauthlogout--json|j$source-coopauthlogout--json|jq-$source-coopauths$source-coopauthst$source-coopauthsta$source-coopauthstat$source-coopauthstatu$source-coopc$source-coopcr$source-coopcre$source-coopcred$source-coopcreds$source-coopcreds;e$source-coopcreds;echo"$source-coopcreds;echo"e$source-coopcreds;echo"ex$source-coopcreds;echo"exi$source-coopcreds;echo"exit$$source-coopcreds;echo"exit$? \ No newline at end of file +$$s$so$sou$sour$sourc$source$source-$source-c$source-co$source-coo$source-coop$source-coopa$source-coopau$source-coopaut$source-coopauth$source-coopauthlogout$source-coopauthlogout--json$source-coopauthlogout--json|$source-coopauthlogout--json|jq$source-coopauthlogout--json|jq-c{"proxy_url":"https://data.source.coop","removed":{"credentials":true,"session":true},"role_arn":"_default"}$source-coopauthstatusEnvironmenthttps://data.source.coopLoginnotloggedinCredentials_default:nonecached$source-coopcreds;$source-coopcreds;ec$source-coopcreds;echo$source-coopcreds;echo"exit$source-coopcreds;echo"exit$?"Error:Notsignedin:run'source-coopauthlogin'.exit4$source-coopauthl$source-coopauthlo$source-coopauthlog$source-coopauthlogo$source-coopauthlogou$source-coopauthlogout-$source-coopauthlogout--$source-coopauthlogout--j$source-coopauthlogout--js$source-coopauthlogout--jso$source-coopauthlogout--json|j$source-coopauthlogout--json|jq-$source-coopauths$source-coopauthst$source-coopauthsta$source-coopauthstat$source-coopauthstatu$source-coopc$source-coopcr$source-coopcre$source-coopcred$source-coopcreds$source-coopcreds;e$source-coopcreds;echo"$source-coopcreds;echo"e$source-coopcreds;echo"ex$source-coopcreds;echo"exi$source-coopcreds;echo"exit$$source-coopcreds;echo"exit$? \ No newline at end of file diff --git a/src/api.rs b/src/api.rs index 0fe86bf..579795c 100644 --- a/src/api.rs +++ b/src/api.rs @@ -223,6 +223,9 @@ impl ApiError { } } +/// The error when there's no login session, for API calls and credentials. +pub const NOT_SIGNED_IN: &str = "Not signed in: run 'source-coop auth login'."; + /// The User-Agent sent to source.coop, so the server can tell CLI traffic, /// and its version, from browsers and curl. pub const USER_AGENT: &str = concat!("source-coop-cli/", env!("CARGO_PKG_VERSION")); @@ -347,9 +350,7 @@ impl Client { /// a member of. pub async fn check_signed_in(&self) -> Result { if self.token.is_none() { - return Err(ApiError::other( - "Not signed in: run 'source-coop auth login'.".into(), - )); + return Err(ApiError::other(NOT_SIGNED_IN.into())); } self.get(self.url(&["whoami"])).await } diff --git a/src/exec.rs b/src/exec.rs index 9c6eaeb..6b41046 100644 --- a/src/exec.rs +++ b/src/exec.rs @@ -20,13 +20,20 @@ pub struct Env { /// Removed whether signed or not. A profile could hold other credentials or /// an endpoint; the env vars already win, but naming one invites confusion /// over which applied. And tools told to ignore configured endpoints would -/// send requests to AWS itself rather than the proxy. +/// send requests to AWS itself rather than the proxy. A general +/// `AWS_ENDPOINT_URL` (LocalStack, say) is read by `object_store` as its S3 +/// endpoint, and could win over the proxy's `AWS_ENDPOINT`. const ALWAYS_REMOVED: &[&str] = &[ "AWS_PROFILE", "AWS_DEFAULT_PROFILE", "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", + "AWS_ENDPOINT_URL", ]; +/// Removed when signing: switches a user may have exported to read public +/// buckets, which would make GDAL and `object_store` skip the credentials. +const SKIP_SIGNING: &[&str] = &["AWS_NO_SIGN_REQUEST", "AWS_SKIP_SIGNATURE"]; + /// Removed when running unsigned: every other way an AWS tool finds keys. /// Keys the proxy doesn't know (anyone's real AWS keys) are refused even for /// public data, and would be sent, if not the secret, to the proxy. @@ -92,12 +99,15 @@ pub fn env(creds: Option<&Credentials>, proxy_url: &str, region_set: bool) -> En } let mut remove = ALWAYS_REMOVED.to_vec(); match creds { - Some(creds) => set.extend([ - ("AWS_ACCESS_KEY_ID", creds.access_key_id.clone()), - ("AWS_SECRET_ACCESS_KEY", creds.secret_access_key.clone()), - ("AWS_SESSION_TOKEN", creds.session_token.clone()), - ("AWS_CREDENTIAL_EXPIRATION", creds.expiration.clone()), - ]), + Some(creds) => { + remove.extend(SKIP_SIGNING); + set.extend([ + ("AWS_ACCESS_KEY_ID", creds.access_key_id.clone()), + ("AWS_SECRET_ACCESS_KEY", creds.secret_access_key.clone()), + ("AWS_SESSION_TOKEN", creds.session_token.clone()), + ("AWS_CREDENTIAL_EXPIRATION", creds.expiration.clone()), + ]); + } None => { remove.extend(OTHER_CREDENTIALS); set.extend([ @@ -125,11 +135,14 @@ pub fn unsigned_hint(command: &[String]) -> Option { if !matches!(name, "aws" | "s5cmd") || args.iter().any(|a| a == "--no-sign-request") { return None; } - let args: Vec<_> = args.iter().map(|a| quoted(a)).collect(); + let words: Vec<_> = std::iter::once(program.as_str()) + .chain(["--no-sign-request"]) + .chain(args.iter().map(String::as_str)) + .map(quoted) + .collect(); Some(format!( - "{name} needs --no-sign-request to run unsigned: \ - source-coop exec -- {program} --no-sign-request {}", - args.join(" ") + "{name} needs --no-sign-request to run unsigned: source-coop exec -- {}", + words.join(" ") )) } @@ -211,6 +224,10 @@ mod tests { None, "only S3 goes to the proxy" ); + assert!( + env.remove.contains(&"AWS_ENDPOINT_URL"), + "a general endpoint can't take object_store's S3 requests" + ); assert_eq!( get(&env, "AWS_S3_ENDPOINT"), Some("data.staging.source.coop") @@ -220,7 +237,8 @@ mod tests { assert_eq!(get(&env, "AWS_ALLOW_HTTP"), None); assert_eq!(get(&env, "AWS_REGION"), Some("us-west-2")); assert_eq!(get(&env, "AWS_DEFAULT_REGION"), Some("us-west-2")); - // Signed: nothing says to skip signing, and only profiles go. + // Signed: nothing says to skip signing, and switches to skip it + // that the user exported are removed along with profiles. for name in [ "AWS_NO_SIGN_REQUEST", "AWS_SKIP_SIGNATURE", @@ -228,7 +246,7 @@ mod tests { ] { assert_eq!(get(&env, name), None, "{name}"); } - assert_eq!(env.remove, ALWAYS_REMOVED); + assert_eq!(env.remove, [ALWAYS_REMOVED, SKIP_SIGNING].concat()); } #[test] @@ -304,6 +322,14 @@ mod tests { None ); assert_eq!(unsigned_hint(&cmd("gdalinfo /vsis3/acct/p/a.tif")), None); + let spaced = vec!["/opt/my tools/aws".to_string(), "s3".into(), "ls".into()]; + assert_eq!( + unsigned_hint(&spaced).as_deref(), + Some( + "aws needs --no-sign-request to run unsigned: \ + source-coop exec -- '/opt/my tools/aws' --no-sign-request s3 ls" + ) + ); } #[test] diff --git a/src/main.rs b/src/main.rs index e7b54a7..3115bee 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1055,8 +1055,9 @@ async fn run_exec(args: ExecArgs, verbose: bool) -> Result<(), String> { EXEC_MIN_LIFETIME, verbose, ) - .await; - let (env, notes) = exec_env(&args.command, &args.env.proxy_url, signed(creds)?); + .await? + .map(|entry| entry.creds); + let (env, notes) = exec_env(&args.command, &args.env.proxy_url, creds); for note in notes { eprintln!("{note}"); } @@ -1069,18 +1070,6 @@ async fn run_exec(args: ExecArgs, verbose: bool) -> Result<(), String> { exec::run(&args.command, env) } -/// The credentials `exec` signs with, or None to run unsigned: only when -/// nobody is signed in. A session that can't be renewed is an error instead, -/// since whoever signed in means to act as themselves, and an unsigned run -/// would turn that into a puzzling refusal of their own data. -fn signed(creds: Result) -> Result, String> { - match creds { - Ok(entry) => Ok(Some(entry.creds)), - Err(e) if e == NOT_SIGNED_IN => Ok(None), - Err(e) => Err(e), - } -} - /// The environment `exec` runs `command` in, and the notes to print first. fn exec_env( command: &[String], @@ -1118,7 +1107,8 @@ async fn run_creds(args: CredsArgs, verbose: bool) -> Result<(), String> { CREDS_MIN_LIFETIME, verbose, ) - .await?; + .await? + .ok_or(api::NOT_SIGNED_IN)?; match args.output.format { OutputFormat::CredentialProcess => output::print_credential_process(&entry.creds), OutputFormat::Env => output::print_env(&entry.creds), @@ -1131,64 +1121,61 @@ async fn run_creds(args: CredsArgs, verbose: bool) -> Result<(), String> { /// The role's credentials at `proxy_url`'s environment: the cached ones while /// they have `min_left` to run, otherwise new ones, minted once however many -/// ask at a time. +/// ask at a time. None when nobody is signed in. A session that can't be +/// renewed is an error, not None: whoever signed in means to act as +/// themselves, and `exec` running unsigned would turn that into a puzzling +/// refusal of their own data. async fn current_creds( role_arn: &str, proxy_url: &str, min_left: chrono::Duration, verbose: bool, -) -> Result { +) -> Result, String> { let env = cache::env(proxy_url)?; let env = env.as_deref(); let fresh = |entry: &cache::CacheEntry| -> Result { Ok(!cache::expires_within(&entry.creds, min_left)?) }; match cache::read_credentials(env, role_arn)? { - Some(entry) if fresh(&entry)? => return Ok(entry), + Some(entry) if fresh(&entry)? => return Ok(Some(entry)), _ => {} } // Re-check under the lock: another process may have minted already. let _lock = cache::lock(env, role_arn)?; match cache::read_credentials(env, role_arn)? { - Some(entry) if fresh(&entry)? => Ok(entry), + Some(entry) if fresh(&entry)? => Ok(Some(entry)), stale => mint(role_arn, proxy_url, env, stale, verbose) .await - .map_err(|e| match e.as_str() { - NOT_LOGGED_IN => NOT_SIGNED_IN.to_string(), - _ => format!("Couldn't get new credentials ({e}). Run 'source-coop auth login'."), + .map_err(|e| { + format!("Couldn't get new credentials ({e}). Run 'source-coop auth login'.") }), } } -/// `mint`'s error when there's no login session to mint with. -const NOT_LOGGED_IN: &str = "not logged in"; - -/// `current_creds`'s error when there's no login session. -const NOT_SIGNED_IN: &str = "Not signed in. Run 'source-coop auth login'."; - /// New credentials for the role, cached before they're returned. An entry /// from before the session was shared refreshes with its own token; anything /// else exchanges the session's ID token, refreshing the session if need be. +/// None when there's no login session to mint with. async fn mint( role_arn: &str, proxy_url: &str, env: Option<&str>, stale: Option, verbose: bool, -) -> Result { +) -> Result, String> { let save = |e: &cache::CacheEntry| cache::write_credentials(env, role_arn, e).map(drop); match stale { Some(entry) if entry.sts.is_none() && entry.refresh.is_some() => { - refresh(role_arn, entry, verbose, save).await + refresh(role_arn, entry, verbose, save).await.map(Some) } stale => { let sts = stale.and_then(|e| e.sts).unwrap_or(cache::StsSettings { proxy_url: proxy_url.to_string(), duration: None, }); - let id_token = session::token(env, session::Want::Id, verbose) - .await? - .ok_or(NOT_LOGGED_IN)?; + let Some(id_token) = session::token(env, session::Want::Id, verbose).await? else { + return Ok(None); + }; let creds = sts::assume_role(&sts.proxy_url, role_arn, &id_token, sts.duration, verbose) .await?; @@ -1198,7 +1185,7 @@ async fn mint( refresh: None, }; save(&entry)?; - Ok(entry) + Ok(Some(entry)) } } } @@ -1816,26 +1803,6 @@ mod tests { assert!(broken.is_empty(), "{}", broken.join("\n\n")); } - #[test] - fn exec_runs_unsigned_only_when_nobody_is_signed_in() { - let creds = sts::Credentials { - access_key_id: "AKID".into(), - secret_access_key: "SECRET".into(), - session_token: "TOKEN".into(), - expiration: "2026-10-14T13:00:00Z".into(), - }; - let entry = cache::CacheEntry { - creds: creds.clone(), - sts: None, - refresh: None, - }; - assert_eq!(signed(Ok(entry)), Ok(Some(creds))); - assert_eq!(signed(Err(NOT_SIGNED_IN.into())), Ok(None)); - // Signed in, but the session can't be renewed: an error, not unsigned. - let refused = "Couldn't get new credentials (invalid_grant). Run 'source-coop auth login'."; - assert_eq!(signed(Err(refused.into())), Err(refused.into())); - } - #[test] fn exec_says_when_it_runs_unsigned() { let cmd = |s: &str| s.split(' ').map(String::from).collect::>();