diff --git a/core/Cargo.lock b/core/Cargo.lock index dc789d9d1543..73ffbd0a23ed 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -6443,7 +6443,7 @@ dependencies = [ "mea", "percent-encoding", "pretty_assertions", - "quick-xml 0.41.0", + "quick-xml", "rand 0.10.1", "reqsign-core", "serde", @@ -6805,7 +6805,7 @@ dependencies = [ "opendal-core", "opendal-service-azure-common", "pretty_assertions", - "quick-xml 0.41.0", + "quick-xml", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -6827,7 +6827,7 @@ dependencies = [ "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.41.0", + "quick-xml", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -6845,7 +6845,7 @@ dependencies = [ "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.41.0", + "quick-xml", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -6916,7 +6916,7 @@ dependencies = [ "http 1.4.2", "log", "opendal-core", - "quick-xml 0.41.0", + "quick-xml", "reqsign-core", "reqsign-file-read-tokio", "reqsign-tencent-cos", @@ -7049,7 +7049,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.41.0", + "quick-xml", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -7313,7 +7313,7 @@ dependencies = [ "http 1.4.2", "log", "opendal-core", - "quick-xml 0.41.0", + "quick-xml", "reqsign-core", "reqsign-file-read-tokio", "reqsign-huaweicloud-obs", @@ -7358,7 +7358,7 @@ dependencies = [ "log", "opendal-core", "pretty_assertions", - "quick-xml 0.41.0", + "quick-xml", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -7445,12 +7445,14 @@ dependencies = [ "md-5 0.11.0", "opendal-core", "pretty_assertions", - "quick-xml 0.41.0", + "quick-xml", "reqsign-aws-v4", + "reqsign-command-execute-tokio", "reqsign-core", "reqsign-file-read-tokio", "serde", "serde_json", + "tempfile", "tokio", "url", ] @@ -7528,7 +7530,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.41.0", + "quick-xml", "serde", "serde_json", "sha1 0.11.0", @@ -7555,7 +7557,7 @@ dependencies = [ "bytes", "http 1.4.2", "opendal-core", - "quick-xml 0.41.0", + "quick-xml", "reqsign-core", "reqsign-file-read-tokio", "reqsign-volcengine-tos", @@ -7574,7 +7576,7 @@ dependencies = [ "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.41.0", + "quick-xml", "serde", "serde_json", "sha1 0.11.0", @@ -7603,7 +7605,7 @@ dependencies = [ "http 1.4.2", "log", "opendal-core", - "quick-xml 0.41.0", + "quick-xml", "serde", "serde_json", "tokio", @@ -7619,7 +7621,7 @@ dependencies = [ "log", "mea", "opendal-core", - "quick-xml 0.41.0", + "quick-xml", "serde", "tokio", ] @@ -7648,7 +7650,7 @@ dependencies = [ "http 1.4.2", "log", "opendal-core", - "quick-xml 0.41.0", + "quick-xml", "serde", "serde_json", "tokio", @@ -8718,16 +8720,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "quick-xml" -version = "0.40.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick-xml" version = "0.41.0" @@ -9160,9 +9152,8 @@ dependencies = [ [[package]] name = "reqsign-aliyun-oss" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372266b4733756738eeb199a98188037d27a0989980e2600ae7ce1faf00a867d" +version = "3.1.2" +source = "git+https://github.com/apache/opendal-reqsign?rev=478f4c42713ab90e8a817e0fc4fa0d7469bfce06#478f4c42713ab90e8a817e0fc4fa0d7469bfce06" dependencies = [ "anyhow", "form_urlencoded", @@ -9177,9 +9168,8 @@ dependencies = [ [[package]] name = "reqsign-aws-v4" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75624bd8a466e37ddc0a7b6c33ac859a85347c153a916e1dd9d0b68338f74a" +version = "3.0.3" +source = "git+https://github.com/apache/opendal-reqsign?rev=478f4c42713ab90e8a817e0fc4fa0d7469bfce06#478f4c42713ab90e8a817e0fc4fa0d7469bfce06" dependencies = [ "anyhow", "bytes", @@ -9188,7 +9178,7 @@ dependencies = [ "http 1.4.2", "log", "percent-encoding", - "quick-xml 0.40.1", + "quick-xml", "reqsign-core", "rust-ini", "serde", @@ -9199,9 +9189,8 @@ dependencies = [ [[package]] name = "reqsign-azure-storage" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b96928e73ad984de1d99e382749d09e5dab7dd707b767974f7e40aa926b82f" +version = "3.1.1" +source = "git+https://github.com/apache/opendal-reqsign?rev=478f4c42713ab90e8a817e0fc4fa0d7469bfce06#478f4c42713ab90e8a817e0fc4fa0d7469bfce06" dependencies = [ "anyhow", "base64 0.22.1", @@ -9218,16 +9207,23 @@ dependencies = [ "sha1 0.11.0", ] +[[package]] +name = "reqsign-command-execute-tokio" +version = "3.0.3" +source = "git+https://github.com/apache/opendal-reqsign?rev=478f4c42713ab90e8a817e0fc4fa0d7469bfce06#478f4c42713ab90e8a817e0fc4fa0d7469bfce06" +dependencies = [ + "reqsign-core", + "tokio", +] + [[package]] name = "reqsign-core" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fa5cb48808693614d1701fcd3db0b30fa292e0f18e122ae068b6d32eaeed3f" +version = "3.2.0" +source = "git+https://github.com/apache/opendal-reqsign?rev=478f4c42713ab90e8a817e0fc4fa0d7469bfce06#478f4c42713ab90e8a817e0fc4fa0d7469bfce06" dependencies = [ "anyhow", "base64 0.22.1", "bytes", - "form_urlencoded", "futures", "hex", "hmac 0.13.0", @@ -9245,9 +9241,8 @@ dependencies = [ [[package]] name = "reqsign-file-read-tokio" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a4b6f3a3fd29ffcc99a90aec585a65217783badfd73acddf847b63ae683bda9" +version = "3.0.3" +source = "git+https://github.com/apache/opendal-reqsign?rev=478f4c42713ab90e8a817e0fc4fa0d7469bfce06#478f4c42713ab90e8a817e0fc4fa0d7469bfce06" dependencies = [ "anyhow", "reqsign-core", @@ -9256,9 +9251,8 @@ dependencies = [ [[package]] name = "reqsign-google" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb215d0876a18b6bd9cdd380b589e5292aaa638ca15266de794b1122d898b6b2" +version = "3.0.3" +source = "git+https://github.com/apache/opendal-reqsign?rev=478f4c42713ab90e8a817e0fc4fa0d7469bfce06#478f4c42713ab90e8a817e0fc4fa0d7469bfce06" dependencies = [ "form_urlencoded", "http 1.4.2", @@ -9274,9 +9268,8 @@ dependencies = [ [[package]] name = "reqsign-huaweicloud-obs" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1136b31eb7202016d8bbb9445bc3e9ee81e7040736f0dae749a8b2b31d738c9" +version = "3.0.3" +source = "git+https://github.com/apache/opendal-reqsign?rev=478f4c42713ab90e8a817e0fc4fa0d7469bfce06#478f4c42713ab90e8a817e0fc4fa0d7469bfce06" dependencies = [ "anyhow", "http 1.4.2", @@ -9287,9 +9280,8 @@ dependencies = [ [[package]] name = "reqsign-tencent-cos" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84110aabba799fbcd48b3abb51fbbff4749f879252e5806b6f5d0cbe0fef6abb" +version = "3.0.3" +source = "git+https://github.com/apache/opendal-reqsign?rev=478f4c42713ab90e8a817e0fc4fa0d7469bfce06#478f4c42713ab90e8a817e0fc4fa0d7469bfce06" dependencies = [ "anyhow", "http 1.4.2", @@ -9302,9 +9294,8 @@ dependencies = [ [[package]] name = "reqsign-volcengine-tos" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d083a363b3577f519ce8425bb50f902622a28a83f7c4a26a5c990b66ec75b3" +version = "3.0.3" +source = "git+https://github.com/apache/opendal-reqsign?rev=478f4c42713ab90e8a817e0fc4fa0d7469bfce06#478f4c42713ab90e8a817e0fc4fa0d7469bfce06" dependencies = [ "anyhow", "http 1.4.2", diff --git a/core/Cargo.toml b/core/Cargo.toml index ab3dbf8b8f20..164e2b8042ba 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -59,6 +59,19 @@ tokio = { version = "1.52", default-features = false } url = "2.5" uuid = { version = "1", default-features = false } +[patch.crates-io] +# Keep reqsign crates on one commit until the profile-selection APIs are formally released. +reqsign-aliyun-oss = { git = "https://github.com/apache/opendal-reqsign", rev = "478f4c42713ab90e8a817e0fc4fa0d7469bfce06" } +reqsign-aws-v4 = { git = "https://github.com/apache/opendal-reqsign", rev = "478f4c42713ab90e8a817e0fc4fa0d7469bfce06" } +reqsign-azure-storage = { git = "https://github.com/apache/opendal-reqsign", rev = "478f4c42713ab90e8a817e0fc4fa0d7469bfce06" } +reqsign-command-execute-tokio = { git = "https://github.com/apache/opendal-reqsign", rev = "478f4c42713ab90e8a817e0fc4fa0d7469bfce06" } +reqsign-core = { git = "https://github.com/apache/opendal-reqsign", rev = "478f4c42713ab90e8a817e0fc4fa0d7469bfce06" } +reqsign-file-read-tokio = { git = "https://github.com/apache/opendal-reqsign", rev = "478f4c42713ab90e8a817e0fc4fa0d7469bfce06" } +reqsign-google = { git = "https://github.com/apache/opendal-reqsign", rev = "478f4c42713ab90e8a817e0fc4fa0d7469bfce06" } +reqsign-huaweicloud-obs = { git = "https://github.com/apache/opendal-reqsign", rev = "478f4c42713ab90e8a817e0fc4fa0d7469bfce06" } +reqsign-tencent-cos = { git = "https://github.com/apache/opendal-reqsign", rev = "478f4c42713ab90e8a817e0fc4fa0d7469bfce06" } +reqsign-volcengine-tos = { git = "https://github.com/apache/opendal-reqsign", rev = "478f4c42713ab90e8a817e0fc4fa0d7469bfce06" } + [package] exclude = ["/tests/"] keywords = ["storage", "fs", "s3", "azblob", "gcs"] diff --git a/core/services/s3/Cargo.toml b/core/services/s3/Cargo.toml index f867d69aec0d..214ea20e500b 100644 --- a/core/services/s3/Cargo.toml +++ b/core/services/s3/Cargo.toml @@ -42,6 +42,7 @@ opendal-core = { path = "../../core", version = "0.58.1", default-features = fal ] } quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] } reqsign-aws-v4 = { version = "3.0.0", default-features = false } +reqsign-command-execute-tokio = { version = "3.0.1", default-features = false } reqsign-core = { version = "3.0.1", default-features = false } reqsign-file-read-tokio = { version = "3.0.1", default-features = false } serde = { workspace = true, features = ["derive"] } @@ -50,4 +51,5 @@ url = { workspace = true } [dev-dependencies] pretty_assertions = "1" serde_json = { workspace = true } +tempfile = "3" tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/core/services/s3/src/backend.rs b/core/services/s3/src/backend.rs index 9d0869789371..d3ac25129b5a 100644 --- a/core/services/s3/src/backend.rs +++ b/core/services/s3/src/backend.rs @@ -35,6 +35,7 @@ use reqsign_aws_v4::Credential; use reqsign_aws_v4::DefaultCredentialProvider; use reqsign_aws_v4::RequestSigner as AwsV4Signer; use reqsign_aws_v4::StaticCredentialProvider; +use reqsign_command_execute_tokio::TokioCommandExecute; use reqsign_core::Context; use reqsign_core::OsEnv; use reqsign_core::ProvideCredentialChain; @@ -146,6 +147,22 @@ impl S3Builder { self } + /// Set the AWS profile used by the default credential provider chain. + /// + /// The configured profile takes precedence over the `AWS_PROFILE` + /// environment variable and applies to shared AWS config and credentials + /// files, SSO, and `credential_process`. + /// + /// This setting has no effect when [`Self::disable_config_load`] is set or + /// when [`Self::credential_provider_chain`] replaces the default chain. + pub fn profile(mut self, profile: &str) -> Self { + if !profile.is_empty() { + self.config.profile = Some(profile.to_string()) + } + + self + } + /// Set access_key_id of this backend. /// /// - If access_key_id is set, we will take user's input first. @@ -481,6 +498,26 @@ impl S3Builder { self } + fn default_credential_provider(config: &S3Config) -> DefaultCredentialProvider { + let mut builder = DefaultCredentialProvider::builder(); + + if config.disable_config_load { + builder = builder.no_env().no_profile(); + } else if let Some(profile) = config + .profile + .as_deref() + .filter(|profile| !profile.is_empty()) + { + builder = builder.with_profile(profile); + } + + if config.disable_ec2_metadata { + builder = builder.no_imds(); + } + + builder.build() + } + /// Check if `bucket` is valid. /// `bucket` must be not empty and if `enable_virtual_host_style` is true /// it could not contain dot (.) character. @@ -842,24 +879,13 @@ impl Builder for S3Builder { let endpoint = Self::build_endpoint(&config, ®ion); debug!("backend use endpoint: {endpoint}"); - // The base signer context only carries local config readers. HTTP - // sending is injected from OperationContext when S3Core signs each - // operation. - let ctx = Context::new().with_file_read(TokioFileRead).with_env(OsEnv); + let ctx = Context::new() + .with_file_read(TokioFileRead) + .with_env(OsEnv) + .with_command_execute(TokioCommandExecute); - let mut provider = { - let mut builder = DefaultCredentialProvider::builder(); - - if config.disable_config_load { - builder = builder.no_env().no_profile(); - } - - if config.disable_ec2_metadata { - builder = builder.no_imds(); - } - - ProvideCredentialChain::new().push(builder.build()) - }; + let mut provider = + ProvideCredentialChain::new().push(Self::default_credential_provider(&config)); // Insert static key if user provided. if let (Some(ak), Some(sk)) = (&config.access_key_id, &config.secret_access_key) { @@ -1232,6 +1258,13 @@ impl Service for S3Backend { } #[cfg(test)] mod tests { + use std::collections::HashMap; + use std::fs; + + use reqsign_core::ProvideCredential; + use reqsign_core::StaticEnv; + use tempfile::tempdir; + use super::*; #[test] @@ -1287,6 +1320,68 @@ mod tests { } } + #[tokio::test] + async fn test_profile_selects_default_credential_provider() { + let temp_dir = tempdir().expect("create temporary directory"); + let credentials_file = temp_dir.path().join("credentials"); + fs::write( + &credentials_file, + "\ +[ambient] +aws_access_key_id = AMBIENT +aws_secret_access_key = ambient-secret + +[selected] +aws_access_key_id = SELECTED +aws_secret_access_key = selected-secret +", + ) + .expect("write credentials file"); + + let context = Context::new() + .with_file_read(TokioFileRead) + .with_env(StaticEnv { + home_dir: None, + envs: HashMap::from([ + ("AWS_PROFILE".to_string(), "ambient".to_string()), + ( + "AWS_SHARED_CREDENTIALS_FILE".to_string(), + credentials_file.to_string_lossy().into_owned(), + ), + ]), + }); + + let selected = S3Builder::default() + .profile("selected") + .disable_ec2_metadata(); + let credential = S3Builder::default_credential_provider(&selected.config) + .provide_credential(&context) + .await + .expect("load selected profile") + .expect("selected profile must provide credentials"); + assert_eq!(credential.access_key_id, "SELECTED"); + assert_eq!(credential.secret_access_key, "selected-secret"); + + let ambient = S3Builder::default().disable_ec2_metadata(); + let credential = S3Builder::default_credential_provider(&ambient.config) + .provide_credential(&context) + .await + .expect("load ambient profile") + .expect("ambient profile must provide credentials"); + assert_eq!(credential.access_key_id, "AMBIENT"); + assert_eq!(credential.secret_access_key, "ambient-secret"); + + let disabled = S3Builder::default() + .profile("selected") + .disable_config_load() + .disable_ec2_metadata(); + let credential = S3Builder::default_credential_provider(&disabled.config) + .provide_credential(&context) + .await + .expect("disabled config loading must not fail"); + assert!(credential.is_none()); + } + #[tokio::test] async fn test_detect_region() { let cases = vec![ diff --git a/core/services/s3/src/config.rs b/core/services/s3/src/config.rs index 0b3a7cafee85..79f053191dfc 100644 --- a/core/services/s3/src/config.rs +++ b/core/services/s3/src/config.rs @@ -94,6 +94,17 @@ pub struct S3Config { #[serde(alias = "aws_region")] pub region: Option, + /// AWS profile used by the default credential provider chain. + /// + /// This profile takes precedence over the `AWS_PROFILE` environment + /// variable and applies to shared AWS config and credentials files, SSO, + /// and `credential_process`. + /// + /// + /// + #[serde(alias = "aws_profile")] + pub profile: Option, + /// access_key_id of this backend. /// /// - If access_key_id is set, we will take user's input first. @@ -367,6 +378,7 @@ mod tests { "secret_access_key": "test-secret", "region": "us-west-2", "endpoint": "https://s3.amazonaws.com", + "profile": "development", "session_token": "test-token" }"#; @@ -379,6 +391,7 @@ mod tests { config.endpoint, Some("https://s3.amazonaws.com".to_string()) ); + assert_eq!(config.profile, Some("development".to_string())); assert_eq!(config.session_token, Some("test-token".to_string())); } @@ -390,6 +403,7 @@ mod tests { "aws_secret_access_key": "test-secret", "aws_region": "us-west-2", "aws_endpoint": "https://s3.amazonaws.com", + "aws_profile": "staging", "aws_session_token": "test-token" }"#; @@ -402,6 +416,7 @@ mod tests { config.endpoint, Some("https://s3.amazonaws.com".to_string()) ); + assert_eq!(config.profile, Some("staging".to_string())); assert_eq!(config.session_token, Some("test-token".to_string())); } diff --git a/core/services/s3/src/core.rs b/core/services/s3/src/core.rs index a2278a19b4d2..dab923d536bb 100644 --- a/core/services/s3/src/core.rs +++ b/core/services/s3/src/core.rs @@ -40,7 +40,8 @@ use http::header::IF_MODIFIED_SINCE; use http::header::IF_NONE_MATCH; use http::header::IF_UNMODIFIED_SINCE; use reqsign_aws_v4::Credential; -use reqsign_core::{Context, Signer}; +use reqsign_command_execute_tokio::TokioCommandExecute; +use reqsign_core::{Context, OsEnv, Signer}; use serde::Deserialize; use serde::Serialize; @@ -141,7 +142,8 @@ impl S3Core { Context::new() .with_file_read(reqsign_file_read_tokio::TokioFileRead) .with_http_send(ctx.http_transport().clone()) - .with_env(reqsign_core::OsEnv), + .with_env(OsEnv) + .with_command_execute(TokioCommandExecute), ) } diff --git a/core/services/s3/src/docs.md b/core/services/s3/src/docs.md index 45788b2d6860..ead26e412671 100644 --- a/core/services/s3/src/docs.md +++ b/core/services/s3/src/docs.md @@ -18,6 +18,7 @@ This service can be used to: - `bucket`: Set the container name for backend. - `endpoint`: Set the endpoint for backend. - `region`: Set the region for backend. +- `profile`: Set the AWS profile for the default credential provider chain. - `access_key_id`: Set the access_key_id for backend. - `secret_access_key`: Set the secret_access_key for backend. - `session_token`: Set the session_token for backend.