From 47e8aa5add8cb6845a219758ab2f54a90e706524 Mon Sep 17 00:00:00 2001 From: Jarrett Tierney Date: Mon, 6 May 2024 16:07:09 -0700 Subject: [PATCH 1/3] os: remove conditional compilation --- packages/os/os.spec | 70 +++++++++++++++------------------------------ 1 file changed, 23 insertions(+), 47 deletions(-) diff --git a/packages/os/os.spec b/packages/os/os.spec index 102c6b6e0c6..3c1c24317c6 100644 --- a/packages/os/os.spec +++ b/packages/os/os.spec @@ -96,18 +96,13 @@ Requires: %{_cross_os}thar-be-settings Requires: %{_cross_os}thar-be-updates Requires: %{_cross_os}updog -%if %{with aws_k8s_family} -Requires: %{_cross_os}pluto -%endif +Requires: (%{_cross_os}pluto if %{_cross_os}variant-family(aws-k8s)) +Requires: (%{_cross_os}shibaken if %{_cross_os}variant-platform(aws)) +Requires: (%{_cross_os}cfsignal if %{_cross_os}variant-platform(aws)) -%if %{with aws_platform} -Requires: %{_cross_os}shibaken -Requires: %{_cross_os}cfsignal -%endif +Requires: (%{_cross_os}warm-pool-wait if %{_cross_os}variant-family(aws-k8s)) -%if %{with nvidia_flavor} -Requires: %{_cross_os}driverdog -%endif +Requires: (%{_cross_os}driverdog if %{_cross_os}variant-flavor(nvidia)) %description %{summary}. @@ -214,24 +209,26 @@ Summary: Bottlerocket certificates handler %description -n %{_cross_os}certdog %{summary}. -%if %{with aws_k8s_family} %package -n %{_cross_os}pluto Summary: Dynamic setting generator for kubernetes %description -n %{_cross_os}pluto %{summary}. -%endif -%if %{with aws_platform} %package -n %{_cross_os}shibaken Summary: Run tasks reliant on IMDS %description -n %{_cross_os}shibaken %{summary}. +%package -n %{_cross_os}warm-pool-wait +Summary: Warm pool wait for aws k8s +Requires: %{_cross_os}shibaken +%description -n %{_cross_os}warm-pool-wait +%{summary}. + %package -n %{_cross_os}cfsignal Summary: Bottlerocket CloudFormation Stack signaler %description -n %{_cross_os}cfsignal %{summary}. -%endif %package -n %{_cross_os}shimpei Summary: OCI-compatible shim around oci-add-hooks @@ -239,13 +236,11 @@ Requires: %{_cross_os}oci-add-hooks %description -n %{_cross_os}shimpei %{summary}. -%if %{with nvidia_flavor} %package -n %{_cross_os}driverdog Summary: Tool to load additional drivers Requires: %{_cross_os}binutils %description -n %{_cross_os}driverdog %{summary}. -%endif %package -n %{_cross_os}bootstrap-containers Summary: Manages bootstrap-containers @@ -319,7 +314,6 @@ exec 1>"${static_output}" 2>&1 static_pid="$!" exec 1>&3 2>&4 -%if %{with aws_platform} || %{with aws_k8s_family} # The AWS SDK crates are extremely slow to build with only one codegen unit. # Pessimize the release build for just the crates that depend on them. # Store the output so we can print it after waiting for the backgrounded job. @@ -333,13 +327,12 @@ CARGO_TARGET_DIR="${HOME}/.cache/.aws-sdk" \ %{__cargo_cross_opts} \ --release \ --manifest-path %{_builddir}/sources/Cargo.toml \ - %{?with_aws_platform: -p cfsignal} \ - %{?with_aws_k8s_family: -p pluto} \ + -p pluto \ + -p cfsignal \ & # Save the PID so we can wait for it later. aws_sdk_pid="$!" exec 1>&3 2>&4 -%endif # Run non-static builds in the foreground. echo "** Output from non-static builds:" @@ -366,8 +359,8 @@ echo "** Output from non-static builds:" -p shimpei \ -p bloodhound \ -p xfscli \ - %{?with_aws_platform: -p shibaken} \ - %{?with_nvidia_flavor: -p driverdog} \ + -p shibaken \ + -p driverdog \ %{nil} # Wait for static builds from the background, if they're not already done. @@ -378,7 +371,6 @@ if [ "${static_rc}" -ne 0 ]; then exit "${static_rc}" fi -%if %{with aws_platform} || %{with aws_k8s_family} # Wait for AWS SDK builds from the background, if they're not already done. set +e; wait "${aws_sdk_pid}"; aws_sdk_rc="${?}"; set -e echo -e "\n** Output from AWS SDK builds:" @@ -386,7 +378,6 @@ cat "${aws_sdk_output}" if [ "${aws_sdk_rc}" -ne 0 ]; then exit "${aws_sdk_rc}" fi -%endif %install install -d %{buildroot}%{_cross_bindir} @@ -402,20 +393,18 @@ for p in \ bottlerocket-cis-checks \ bottlerocket-fips-checks \ kubernetes-cis-checks \ - %{?with_aws_platform: shibaken} \ - %{?with_nvidia_flavor: driverdog} \ + shibaken \ + driverdog \ ; do install -p -m 0755 ${HOME}/.cache/%{__cargo_target}/release/${p} %{buildroot}%{_cross_bindir} done -%if %{with aws_platform} || %{with aws_k8s_family} for p in \ - %{?with_aws_platform: cfsignal} \ - %{?with_aws_k8s_family: pluto} \ + pluto \ + cfsignal \ ; do install -p -m 0755 ${HOME}/.cache/.aws-sdk/%{__cargo_target}/release/${p} %{buildroot}%{_cross_bindir} done -%endif install -d %{buildroot}%{_cross_sbindir} for p in \ @@ -488,10 +477,8 @@ install -d %{buildroot}%{_cross_datadir}/bottlerocket install -d %{buildroot}%{_cross_sysusersdir} install -p -m 0644 %{S:2} %{buildroot}%{_cross_sysusersdir}/api.conf -%if %{with aws_k8s_family} install -d %{buildroot}%{_cross_datadir}/eks install -p -m 0644 %{S:3} %{buildroot}%{_cross_datadir}/eks -%endif install -d %{buildroot}%{_cross_datadir}/updog install -p -m 0644 %{_cross_repo_root_json} %{buildroot}%{_cross_datadir}/updog @@ -504,26 +491,21 @@ install -d %{buildroot}%{_cross_unitdir} install -p -m 0644 \ %{S:100} %{S:102} %{S:103} %{S:105} \ %{S:106} %{S:107} %{S:110} %{S:111} %{S:112} \ - %{S:113} %{S:114} %{S:119} %{S:122} \ + %{S:113} %{S:114} %{S:119} %{S:122} %{S:123} \ %{buildroot}%{_cross_unitdir} -%if %{with nvidia_flavor} sed -e 's|PREFIX|%{_cross_prefix}|g' %{S:115} > link-kernel-modules.service sed -e 's|PREFIX|%{_cross_prefix}|g' %{S:116} > load-kernel-modules.service install -p -m 0644 \ link-kernel-modules.service \ load-kernel-modules.service \ %{buildroot}%{_cross_unitdir} -%endif -%if %{with aws_platform} -%if %{with aws_k8s_family} install -p -m 0644 %{S:10} %{buildroot}%{_cross_templatedir} install -p -m 0644 %{S:120} %{buildroot}%{_cross_unitdir} -%endif + install -p -m 0644 %{S:9} %{buildroot}%{_cross_templatedir} install -p -m 0644 %{S:117} %{buildroot}%{_cross_unitdir} -%endif install -d %{buildroot}%{_cross_tmpfilesdir} install -p -m 0644 %{S:200} %{buildroot}%{_cross_tmpfilesdir}/migration.conf @@ -633,35 +615,29 @@ install -p -m 0644 %{S:400} %{S:401} %{S:402} %{buildroot}%{_cross_licensedir} %files -n %{_cross_os}logdog %{_cross_bindir}/logdog -%if %{with aws_platform} %files -n %{_cross_os}shibaken %{_cross_bindir}/shibaken %dir %{_cross_templatedir} -%if %{with aws_k8s_family} + +%files -n %{_cross_os}warm-pool-wait %{_cross_templatedir}/warm-pool-wait-toml %{_cross_unitdir}/warm-pool-wait.service -%endif %files -n %{_cross_os}cfsignal %{_cross_bindir}/cfsignal %dir %{_cross_templatedir} %{_cross_templatedir}/cfsignal-toml %{_cross_unitdir}/cfsignal.service -%endif -%if %{with nvidia_flavor} %files -n %{_cross_os}driverdog %{_cross_bindir}/driverdog %{_cross_unitdir}/link-kernel-modules.service %{_cross_unitdir}/load-kernel-modules.service -%endif -%if %{with aws_k8s_family} %files -n %{_cross_os}pluto %{_cross_bindir}/pluto %dir %{_cross_datadir}/eks %{_cross_datadir}/eks/eni-max-pods -%endif %files -n %{_cross_os}shimpei %{_cross_bindir}/shimpei From 90654f5693eb0371acfbf17dc8b9bd4a90a4830d Mon Sep 17 00:00:00 2001 From: Jarrett Tierney Date: Mon, 6 May 2024 16:33:56 -0700 Subject: [PATCH 2/3] pluto: convert to be an agent during boot Migrates pluto to fetch a view of required settings from the api using apiclient to generated needed settings for k*s on aws --- packages/os/os.spec | 2 + packages/os/pluto.service | 20 ++ sources/Cargo.lock | 7 +- sources/Cargo.toml | 8 +- sources/api/pluto/Cargo.toml | 7 +- sources/api/pluto/build.rs | 4 - sources/api/pluto/src/api.rs | 215 +++++++++++---- sources/api/pluto/src/aws.rs | 17 +- sources/api/pluto/src/ec2.rs | 17 +- sources/api/pluto/src/eks.rs | 12 +- sources/api/pluto/src/main.rs | 246 ++++++++++-------- sources/api/pluto/src/proxy.rs | 16 -- ...ubernetes-aws-external-cloud-provider.toml | 6 - .../shared-defaults/kubernetes-aws.toml | 4 - 14 files changed, 337 insertions(+), 244 deletions(-) create mode 100644 packages/os/pluto.service diff --git a/packages/os/os.spec b/packages/os/os.spec index 3c1c24317c6..1b6750690c8 100644 --- a/packages/os/os.spec +++ b/packages/os/os.spec @@ -54,6 +54,7 @@ Source117: cfsignal.service Source119: reboot-if-required.service Source120: warm-pool-wait.service Source122: has-boot-ever-succeeded.service +Source123: pluto.service # 2xx sources: tmpfilesd configs Source200: migration-tmpfiles.conf @@ -636,6 +637,7 @@ install -p -m 0644 %{S:400} %{S:401} %{S:402} %{buildroot}%{_cross_licensedir} %files -n %{_cross_os}pluto %{_cross_bindir}/pluto +%{_cross_unitdir}/pluto.service %dir %{_cross_datadir}/eks %{_cross_datadir}/eks/eni-max-pods diff --git a/packages/os/pluto.service b/packages/os/pluto.service new file mode 100644 index 00000000000..1404d780a18 --- /dev/null +++ b/packages/os/pluto.service @@ -0,0 +1,20 @@ +[Unit] +Description=Generate additional settings for Kubernetes +After=network-online.target apiserver.service sundog.service +Requires=sundog.service +# We don't want to restart the unit if the network goes offline or apiserver restarts +Wants=network-online.target apiserver.service +# Block manual interactions with this service, since it could leave the system in additional +# unexpected state +RefuseManualStart=true +RefuseManualStop=true + +[Service] +Type=oneshot +ExecStartPre=/usr/bin/settings-committer +ExecStart=/usr/bin/pluto +RemainAfterExit=true +StandardError=journal+console + +[Install] +RequiredBy=preconfigured.target diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 33ae4e4621d..df531f2314d 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -3213,16 +3213,13 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" name = "pluto" version = "0.1.0" dependencies = [ - "apiclient", "aws-config", "aws-sdk-ec2", "aws-sdk-eks", "aws-smithy-runtime", "aws-smithy-types", "aws-types", - "bottlerocket-variant", "bytes", - "constants", "futures-util", "generate-readme", "headers", @@ -3230,7 +3227,9 @@ dependencies = [ "hyper", "hyper-rustls", "imdsclient", - "models", + "log", + "modeled-types", + "serde", "serde_json", "snafu 0.8.2", "tokio", diff --git a/sources/Cargo.toml b/sources/Cargo.toml index 99877fd8613..dc9c72d0b09 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -64,6 +64,10 @@ members = [ "api/migration/migrations/v1.19.3/public-admin-container-v0-11-6", "api/migration/migrations/v1.19.3/aws-control-container-v0-7-10", "api/migration/migrations/v1.19.3/public-control-container-v0-7-10", + "api/migration/migrations/v1.19.5/aws-control-container-v0-7-11", + "api/migration/migrations/v1.19.5/public-control-container-v0-7-11", + "api/migration/migrations/v1.19.5/aws-admin-container-v0-11-7", + "api/migration/migrations/v1.19.5/public-admin-container-v0-11-7", "api/migration/migrations/v1.20.0/prairiedog-config-file-v0-1-0", "api/migration/migrations/v1.20.0/prairiedog-services-cfg-v0-1-0", "api/migration/migrations/v1.20.0/thar-be-updates-config-file-v0-1-0", @@ -80,10 +84,6 @@ members = [ "api/migration/migrations/v1.20.0/add-ntp-default-options-v0-1-0", "api/migration/migrations/v1.20.0/static-pods-add-prefix-v0-1-0", "api/migration/migrations/v1.20.0/static-pods-services-cfg-v0-1-0", - "api/migration/migrations/v1.19.5/aws-control-container-v0-7-11", - "api/migration/migrations/v1.19.5/public-control-container-v0-7-11", - "api/migration/migrations/v1.19.5/aws-admin-container-v0-11-7", - "api/migration/migrations/v1.19.5/public-admin-container-v0-11-7", "api/migration/migrations/v1.20.0/container-runtime-nvidia", "api/migration/migrations/v1.20.0/container-runtime-metadata-nvidia", "api/migration/migrations/v1.20.0/aws-admin-container-v0-11-8", diff --git a/sources/api/pluto/Cargo.toml b/sources/api/pluto/Cargo.toml index 13cb8f19eb2..9e2b0336fbd 100644 --- a/sources/api/pluto/Cargo.toml +++ b/sources/api/pluto/Cargo.toml @@ -10,29 +10,28 @@ build = "build.rs" exclude = ["README.md"] [dependencies] -apiclient = { path = "../apiclient", version = "0.1" } bytes = "1" -constants = { path = "../../constants", version = "0.1" } futures-util = { version = "0.3", default-features = false } headers = "0.3" http = "0.2" hyper = "0.14" hyper-rustls = { version = "0.24", default-features = false, features = ["http2", "native-tokio", "tls12", "logging"] } imdsclient = { path = "../../imdsclient", version = "0.1" } -models = { path = "../../models", version = "0.1" } +modeled-types = { path = "../../models/modeled-types", version = "0.1" } aws-config = "1" aws-sdk-eks = "1" aws-sdk-ec2 = "1" aws-types = "1" aws-smithy-types = "1" aws-smithy-runtime = "1" +serde = { version = "1", features = ["derive"] } serde_json = "1" snafu = "0.8" tokio = { version = "~1.32", default-features = false, features = ["macros", "rt-multi-thread"] } # LTS tokio-retry = "0.3" tokio-rustls = "0.24" url = "2" +log = "0.4.21" [build-dependencies] -bottlerocket-variant = { version = "0.1", path = "../../bottlerocket-variant" } generate-readme = { version = "0.1", path = "../../generate-readme" } diff --git a/sources/api/pluto/build.rs b/sources/api/pluto/build.rs index a2d24d234b2..5b3a661c3f5 100644 --- a/sources/api/pluto/build.rs +++ b/sources/api/pluto/build.rs @@ -1,7 +1,3 @@ -use bottlerocket_variant::Variant; - fn main() { - let variant = Variant::from_env().unwrap(); - variant.emit_cfgs(); generate_readme::from_main().unwrap(); } diff --git a/sources/api/pluto/src/api.rs b/sources/api/pluto/src/api.rs index 052a32e6955..9784f155461 100644 --- a/sources/api/pluto/src/api.rs +++ b/sources/api/pluto/src/api.rs @@ -1,76 +1,177 @@ -pub(super) use inner::{get_aws_k8s_info, Error}; +use serde::{Deserialize, Serialize}; +use snafu::{ensure, ResultExt, Snafu}; +use std::ffi::OsStr; +use tokio::process::Command; /// The result type for the [`api`] module. pub(super) type Result = std::result::Result; +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] pub(crate) struct AwsK8sInfo { + #[serde(skip)] pub(crate) region: Option, + #[serde(skip)] + pub(crate) https_proxy: Option, + #[serde(skip)] + pub(crate) no_proxy: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) cluster_name: Option, - pub(crate) cluster_dns_ip: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) cluster_dns_ip: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) node_ip: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) max_pods: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) provider_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) hostname_override: Option, + #[serde(skip)] + pub(crate) variant_id: String, } -/// This code is the 'actual' implementation compiled when the `sources` workspace is being compiled -/// for `aws-k8s-*` variants. -#[cfg(variant_family = "aws-k8s")] -mod inner { - use super::*; - use snafu::{ResultExt, Snafu}; +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct AwsInfo { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) region: Option, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct Kubernetes { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) cluster_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) cluster_dns_ip: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) node_ip: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) max_pods: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) provider_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) hostname_override: Option, +} - #[derive(Debug, Snafu)] - pub(crate) enum Error { - #[snafu(display("Error calling Bottlerocket API: {}", source))] - ApiClient { - #[snafu(source(from(apiclient::Error, Box::new)))] - source: Box, - uri: String, - }, +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +struct Os { + variant_id: String, +} - #[snafu(display("Unable to deserialize Bottlerocket settings: {}", source))] - SettingsJson { source: serde_json::Error }, - } +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +struct Network { + https_proxy: Option, + no_proxy: Option, +} - /// Gets the Bottlerocket settings from the API and deserializes them into a struct. - async fn get_settings() -> Result { - let uri = constants::API_SETTINGS_URI; - let (_status, response_body) = - apiclient::raw_request(constants::API_SOCKET, uri, "GET", None) - .await - .context(ApiClientSnafu { uri })?; +#[derive(Deserialize)] +struct View { + pub aws: Option, + pub network: Option, + pub kubernetes: Option, +} - serde_json::from_str(&response_body).context(SettingsJsonSnafu) - } +#[derive(Deserialize)] +struct SettingsView { + pub os: Os, + pub settings: View, +} - /// Gets the info that we need to know about the EKS cluster from the Bottlerocket API. - pub(crate) async fn get_aws_k8s_info() -> Result { - let settings = get_settings().await?; - Ok(AwsK8sInfo { - region: settings.aws.and_then(|a| a.region).map(|s| s.into()), - cluster_name: settings - .kubernetes - .as_ref() - .and_then(|k| k.cluster_name.clone()) - .map(|s| s.into()), - cluster_dns_ip: settings.kubernetes.and_then(|k| k.cluster_dns_ip), - }) - } +#[derive(Debug, Snafu)] +pub(crate) enum Error { + #[snafu(display("Failed to call apiclient: {}", source))] + CommandFailure { source: std::io::Error }, + #[snafu(display("apiclient execution failed: {}", reason))] + ExecutionFailure { reason: String }, + #[snafu(display("Deserialization of configuration file failed: {}", source))] + Deserialize { + #[snafu(source(from(serde_json::Error, Box::new)))] + source: Box, + }, } -/// This dummy code is compiled when the `sources` workspace is being compiled for non `aws-k8s-*` -/// variants. -#[cfg(not(variant_family = "aws-k8s"))] -mod inner { - use super::*; - use snafu::Snafu; +pub(crate) async fn client_command(args: I) -> Result> +where + I: IntoIterator, + S: AsRef, +{ + let result = Command::new("/usr/bin/apiclient") + .args(args) + .output() + .await + .context(CommandFailureSnafu)?; + + ensure!( + result.status.success(), + ExecutionFailureSnafu { + reason: String::from_utf8_lossy(&result.stderr) + } + ); + + Ok(result.stdout) +} - #[derive(Debug, Snafu)] - pub(crate) enum Error { - #[snafu(display( - "The get_aws_k8s_info function is only compatible with aws-k8s variants" - ))] - WrongVariant, - } +/// Gets the info that we need to know about the EKS cluster from the Bottlerocket API. +pub(crate) async fn get_aws_k8s_info() -> Result { + let view_str = client_command(&[ + "get", + "os.variant_id", + "settings.aws.region", + "settings.network.http-proxy", + "settings.network.no-proxy", + "settings.kubernetes.cluster-name", + "settings.kubernetes.cluster-dns-ip", + "settings.kubernetes.node-ip", + "settings.kubernetes.max-pods", + "settings.kubernetes.provider-id", + "settings.kubernetes.hostname-override", + ]) + .await?; + let view: SettingsView = + serde_json::from_slice(view_str.as_slice()).context(DeserializeSnafu)?; - pub(crate) async fn get_aws_k8s_info() -> Result { - WrongVariantSnafu.fail() - } + Ok(AwsK8sInfo { + variant_id: view.os.variant_id, + region: view.settings.aws.and_then(|a| a.region), + https_proxy: view + .settings + .network + .as_ref() + .and_then(|n| n.https_proxy.clone()), + no_proxy: view + .settings + .network + .as_ref() + .and_then(|n| n.no_proxy.clone()), + cluster_name: view + .settings + .kubernetes + .as_ref() + .and_then(|k| k.cluster_name.clone()), + cluster_dns_ip: view + .settings + .kubernetes + .as_ref() + .and_then(|k| k.cluster_dns_ip.clone()), + node_ip: view + .settings + .kubernetes + .as_ref() + .and_then(|k| k.node_ip.clone()), + max_pods: view.settings.kubernetes.as_ref().and_then(|k| k.max_pods), + provider_id: view + .settings + .kubernetes + .as_ref() + .and_then(|k| k.provider_id.clone()), + hostname_override: view + .settings + .kubernetes + .as_ref() + .and_then(|k| k.hostname_override.clone()), + }) } diff --git a/sources/api/pluto/src/aws.rs b/sources/api/pluto/src/aws.rs index 748170334e8..ec29560d92b 100644 --- a/sources/api/pluto/src/aws.rs +++ b/sources/api/pluto/src/aws.rs @@ -3,23 +3,12 @@ use aws_config::{imds, BehaviorVersion}; use aws_smithy_types::retry::{RetryConfig, RetryConfigBuilder}; use aws_types::region::Region; use aws_types::SdkConfig; -use snafu::Snafu; use std::time::Duration; // Max request retry attempts; Retry many many times and let the caller decide when to terminate const MAX_ATTEMPTS: u32 = 100; const IMDS_CONNECT_TIMEOUT: Duration = Duration::from_secs(3); -#[derive(Debug, Snafu)] -pub(super) enum Error { - #[snafu(display("Failed to build IMDS client: {}", source))] - SdkImds { - source: imds::client::error::BuildError, - }, -} - -type Result = std::result::Result; - fn sdk_imds_client() -> imds::Client { imds::Client::builder() .max_attempts(MAX_ATTEMPTS) @@ -31,15 +20,15 @@ fn sdk_retry_config() -> RetryConfig { RetryConfigBuilder::new().max_attempts(MAX_ATTEMPTS).build() } -pub(crate) async fn sdk_config(region: &str) -> Result { +pub(crate) async fn sdk_config(region: &str) -> SdkConfig { let provider = DefaultCredentialsChain::builder() .imds_client(sdk_imds_client()) .build() .await; - Ok(aws_config::defaults(BehaviorVersion::v2023_11_09()) + aws_config::defaults(BehaviorVersion::v2023_11_09()) .region(Region::new(region.to_owned())) .credentials_provider(provider) .retry_config(sdk_retry_config()) .load() - .await) + .await } diff --git a/sources/api/pluto/src/ec2.rs b/sources/api/pluto/src/ec2.rs index 27eef22a102..f2d3431a16a 100644 --- a/sources/api/pluto/src/ec2.rs +++ b/sources/api/pluto/src/ec2.rs @@ -1,5 +1,5 @@ use crate::aws::sdk_config; -use crate::{aws, proxy}; +use crate::proxy; use aws_smithy_runtime::client::http::hyper_014::HyperClientBuilder; use aws_smithy_types::error::display::DisplayErrorContext; use snafu::{OptionExt, ResultExt, Snafu}; @@ -36,18 +36,17 @@ pub(super) enum Error { #[snafu(context(false), display("{}", source))] Proxy { source: proxy::Error }, - - #[snafu(context(false), display("{}", source))] - SdkConfig { source: aws::Error }, } type Result = std::result::Result; -pub(super) async fn get_private_dns_name(region: &str, instance_id: &str) -> Result { - // Respect proxy environment variables when making AWS EC2 API requests - let (https_proxy, no_proxy) = proxy::fetch_proxy_env(); - - let config = sdk_config(region).await?; +pub(super) async fn get_private_dns_name( + region: &str, + instance_id: &str, + https_proxy: Option, + no_proxy: Option, +) -> Result { + let config = sdk_config(region).await; let client = if let Some(https_proxy) = https_proxy { let http_connector = proxy::setup_http_client(https_proxy, no_proxy)?; diff --git a/sources/api/pluto/src/eks.rs b/sources/api/pluto/src/eks.rs index 1f461be92b8..fb5d8f8e520 100644 --- a/sources/api/pluto/src/eks.rs +++ b/sources/api/pluto/src/eks.rs @@ -1,5 +1,5 @@ use crate::aws::sdk_config; -use crate::{aws, proxy}; +use crate::proxy; use aws_sdk_eks::types::KubernetesNetworkConfigResponse; use aws_smithy_runtime::client::http::hyper_014::HyperClientBuilder; use snafu::{OptionExt, ResultExt, Snafu}; @@ -27,9 +27,6 @@ pub(super) enum Error { #[snafu(context(false), display("{}", source))] Proxy { source: proxy::Error }, - - #[snafu(context(false), display("{}", source))] - SdkConfig { source: aws::Error }, } type Result = std::result::Result; @@ -39,11 +36,10 @@ type Result = std::result::Result; pub(super) async fn get_cluster_network_config( region: &str, cluster: &str, + https_proxy: Option, + no_proxy: Option, ) -> Result { - // Respect proxy environment variables when making AWS EKS API requests - let (https_proxy, no_proxy) = proxy::fetch_proxy_env(); - - let config = sdk_config(region).await?; + let config = sdk_config(region).await; let client = if let Some(https_proxy) = https_proxy { let http_connector = proxy::setup_http_client(https_proxy, no_proxy)?; diff --git a/sources/api/pluto/src/main.rs b/sources/api/pluto/src/main.rs index 749f3bc0ce3..bc4f64b5505 100644 --- a/sources/api/pluto/src/main.rs +++ b/sources/api/pluto/src/main.rs @@ -38,14 +38,16 @@ mod eks; mod hyper_proxy; mod proxy; +use api::AwsK8sInfo; use imdsclient::ImdsClient; +use modeled_types::KubernetesClusterDnsIp; use snafu::{ensure, OptionExt, ResultExt}; use std::fs::File; use std::io::{BufRead, BufReader}; use std::net::IpAddr; +use std::process; use std::str::FromStr; use std::string::String; -use std::{env, process}; // This is the default DNS unless our CIDR block begins with "10." const DEFAULT_DNS_CLUSTER_IP: &str = "10.100.0.10"; @@ -54,6 +56,8 @@ const DEFAULT_10_RANGE_DNS_CLUSTER_IP: &str = "172.20.0.10"; const ENI_MAX_PODS_PATH: &str = "/usr/share/eks/eni-max-pods"; +const NO_HOSTNAME_VARIANTS: &[&str] = &["aws-k8s-1.23", "aws-k8s-1.24", "aws-k8s-1.25"]; + mod error { use crate::{api, ec2, eks}; use snafu::Snafu; @@ -71,8 +75,11 @@ mod error { #[snafu(display("Missing AWS region"))] AwsRegion, - #[snafu(display("Missing field '{}' in EKS network config response", field))] - MissingNetworkConfig { field: &'static str }, + #[snafu(display("Failed to parse setting {} as u32: {}", setting, source))] + ParseToU32 { + setting: String, + source: std::num::ParseIntError, + }, #[snafu(display("Unable to parse CIDR '{}': {}", cidr, reason))] CidrParse { cidr: String, reason: String }, @@ -86,28 +93,9 @@ mod error { #[snafu(display("IMDS request failed: {}", source))] ImdsRequest { source: imdsclient::Error }, - #[snafu(display("IMDS client failed: {}", source))] - ImdsClient { source: imdsclient::Error }, - #[snafu(display("IMDS request failed: No '{}' found", what))] ImdsNone { what: String }, - #[snafu(display("Error deserializing response into JSON from {}: {}", uri, source))] - ImdsJson { - uri: String, - source: serde_json::error::Error, - }, - - #[snafu(display( - "Error serializing to JSON from command output '{}': {}", - output, - source - ))] - OutputJson { - output: String, - source: serde_json::error::Error, - }, - #[snafu(display("{}", source))] EksError { source: eks::Error }, @@ -120,15 +108,15 @@ mod error { source: std::io::Error, }, - #[snafu(display("Failed to parse setting {} as u32: {}", setting, source))] - ParseToU32 { - setting: String, - source: std::num::ParseIntError, - }, - #[snafu(display("Failed to read line: {}", source))] IoReadLine { source: std::io::Error }, + #[snafu(display("Failed to serialize generated settings: {}", source))] + Serialize { source: serde_json::Error }, + + #[snafu(display("Failed to set generated settings: {}", source))] + SetFailure { source: api::Error }, + #[snafu(display( "Unable to find maximum number of pods supported for instance-type {}", instance_type @@ -141,7 +129,15 @@ use error::PlutoError; type Result = std::result::Result; -async fn get_max_pods(client: &mut ImdsClient) -> Result { +async fn generate_max_pods(client: &mut ImdsClient, aws_k8s_info: &mut AwsK8sInfo) -> Result<()> { + if aws_k8s_info.max_pods.is_some() { + return Ok(()); + } + aws_k8s_info.max_pods = get_max_pods(client).await.ok(); + Ok(()) +} + +async fn get_max_pods(client: &mut ImdsClient) -> Result { let instance_type = client .fetch_instance_type() .await @@ -164,7 +160,8 @@ async fn get_max_pods(client: &mut ImdsClient) -> Result { } let tokens: Vec<_> = line.split_whitespace().collect(); if tokens.len() == 2 && tokens[0] == instance_type { - return Ok(tokens[1].to_string()); + let setting = tokens[1]; + return setting.parse().context(error::ParseToU32Snafu { setting }); } } error::NoInstanceTypeMaxPodsSnafu { instance_type }.fail() @@ -176,27 +173,56 @@ async fn get_max_pods(client: &mut ImdsClient) -> Result { /// If that works, it returns the expected cluster DNS IP address which is obtained by substituting /// `10` for the last octet. If the EKS call is not successful, it falls back to using IMDS MAC CIDR /// blocks to return one of two default addresses. -async fn get_cluster_dns_ip(client: &mut ImdsClient) -> Result { +async fn generate_cluster_dns_ip( + client: &mut ImdsClient, + aws_k8s_info: &mut AwsK8sInfo, +) -> Result<()> { + if aws_k8s_info.cluster_dns_ip.is_some() { + return Ok(()); + } + // Retrieve the kubernetes network configuration for the EKS cluster - if let Ok(aws_k8s_info) = api::get_aws_k8s_info().await.context(error::AwsInfoSnafu) { - if let (Some(region), Some(cluster_name)) = (aws_k8s_info.region, aws_k8s_info.cluster_name) + let ip_addr = if let Some(ip) = get_eks_network_config(aws_k8s_info).await? { + ip.clone() + } else { + // If we were unable to obtain or parse the cidr range from EKS, fallback to one of two default + // values based on the IPv4 cidr range of our primary network interface + get_ipv4_cluster_dns_ip_from_imds_mac(client).await? + }; + + aws_k8s_info.cluster_dns_ip = Some(KubernetesClusterDnsIp::Scalar( + IpAddr::from_str(ip_addr.as_str()).context(error::BadIpSnafu { + ip: ip_addr.clone(), + })?, + )); + Ok(()) +} + +/// Retrieves the ip address from the kubernetes network configuration for the +/// EKS Cluster +async fn get_eks_network_config(aws_k8s_info: &AwsK8sInfo) -> Result> { + if let (Some(region), Some(cluster_name)) = ( + aws_k8s_info.region.as_ref(), + aws_k8s_info.cluster_name.as_ref(), + ) { + if let Ok(config) = eks::get_cluster_network_config( + region, + cluster_name, + aws_k8s_info.https_proxy.clone(), + aws_k8s_info.no_proxy.clone(), + ) + .await + .context(error::EksSnafu) { - if let Ok(config) = eks::get_cluster_network_config(®ion, &cluster_name) - .await - .context(error::EksSnafu) - { - // Derive cluster-dns-ip from the service IPv4 CIDR - if let Some(ipv4_cidr) = config.service_ipv4_cidr { - if let Ok(dns_ip) = get_dns_from_ipv4_cidr(&ipv4_cidr) { - return Ok(dns_ip); - } + // Derive cluster-dns-ip from the service IPv4 CIDR + if let Some(ipv4_cidr) = config.service_ipv4_cidr { + if let Ok(dns_ip) = get_dns_from_ipv4_cidr(&ipv4_cidr) { + return Ok(Some(dns_ip)); } } } } - // If we were unable to obtain or parse the cidr range from EKS, fallback to one of two default - // values based on the IPv4 cidr range of our primary network interface - get_ipv4_cluster_dns_ip_from_imds_mac(client).await + Ok(None) } /// Replicates [this] logic from the EKS AMI: @@ -260,24 +286,20 @@ async fn get_ipv4_cluster_dns_ip_from_imds_mac(client: &mut ImdsClient) -> Resul } /// Gets the IP address that should be associated with the node. -async fn get_node_ip(client: &mut ImdsClient) -> Result { - // Retrieve the user specified cluster DNS IP if it's specified, otherwise use the pluto-generated - // cluster DNS IP - let aws_k8s_info = api::get_aws_k8s_info().await.context(error::AwsInfoSnafu)?; - let configured_cluster_dns_ip = aws_k8s_info +async fn generate_node_ip(client: &mut ImdsClient, aws_k8s_info: &mut AwsK8sInfo) -> Result<()> { + if aws_k8s_info.node_ip.is_some() { + return Ok(()); + } + // Ensure that this was set in case changes to main occur + generate_cluster_dns_ip(client, aws_k8s_info).await?; + let cluster_dns_ip = aws_k8s_info .cluster_dns_ip - .and_then(|cluster_ip| cluster_ip.iter().next().cloned()); - - let cluster_dns_ip = if let Some(ip) = configured_cluster_dns_ip { - ip.to_owned() - } else { - let ip = get_cluster_dns_ip(client).await?; - IpAddr::from_str(ip.as_str()).context(error::BadIpSnafu { ip })? - }; - + .as_ref() + .and_then(|x| x.iter().next()) + .context(error::NoIpSnafu)?; // If the cluster DNS IP is an IPv4 address, retrieve the IPv4 address for the instance. // If the cluster DNS IP is an IPv6 address, retrieve the IPv6 address for the instance. - match cluster_dns_ip { + let node_ip = match cluster_dns_ip { IpAddr::V4(_) => client .fetch_local_ipv4_address() .await @@ -292,11 +314,22 @@ async fn get_node_ip(client: &mut ImdsClient) -> Result { .context(error::ImdsNoneSnafu { what: "ipv6s associated with primary network interface", }), - } + }?; + aws_k8s_info.node_ip = Some(node_ip); + Ok(()) } /// Gets the provider ID that should be associated with the node -async fn get_provider_id(client: &mut ImdsClient) -> Result { +async fn generate_provider_id( + client: &mut ImdsClient, + aws_k8s_info: &mut AwsK8sInfo, +) -> Result<()> { + if aws_k8s_info.provider_id.is_some() + || NO_HOSTNAME_VARIANTS.contains(&aws_k8s_info.variant_id.as_str()) + { + return Ok(()); + } + let instance_id = client .fetch_instance_id() .await @@ -311,14 +344,21 @@ async fn get_provider_id(client: &mut ImdsClient) -> Result { .context(error::ImdsRequestSnafu)? .context(error::ImdsNoneSnafu { what: "zone" })?; - Ok(format!("aws:///{}/{}", zone, instance_id)) + aws_k8s_info.provider_id = Some(format!("aws:///{}/{}", zone, instance_id)); + Ok(()) } -async fn get_private_dns_name(client: &mut ImdsClient) -> Result { - let region = api::get_aws_k8s_info() - .await - .context(error::AwsInfoSnafu)? +async fn generate_private_dns_name( + client: &mut ImdsClient, + aws_k8s_info: &mut AwsK8sInfo, +) -> Result<()> { + if aws_k8s_info.hostname_override.is_some() { + return Ok(()); + } + + let region = aws_k8s_info .region + .as_ref() .context(error::AwsRegionSnafu)?; let instance_id = client .fetch_instance_id() @@ -327,60 +367,38 @@ async fn get_private_dns_name(client: &mut ImdsClient) -> Result { .context(error::ImdsNoneSnafu { what: "instance ID", })?; - ec2::get_private_dns_name(®ion, &instance_id) + aws_k8s_info.hostname_override = Some( + ec2::get_private_dns_name( + region, + &instance_id, + aws_k8s_info.https_proxy.clone(), + aws_k8s_info.no_proxy.clone(), + ) .await - .context(error::Ec2Snafu) -} - -/// Print usage message. -fn usage() -> ! { - let program_name = env::args().next().unwrap_or_else(|| "program".to_string()); - eprintln!( - r"Usage: {} [max-pods | cluster-dns-ip | node-ip | provider-id | private-dns-name]", - program_name + .context(error::Ec2Snafu)?, ); - process::exit(1); -} - -/// Parses args for the setting key name. -fn parse_args(mut args: env::Args) -> String { - args.nth(1).unwrap_or_else(|| usage()) + Ok(()) } async fn run() -> Result<()> { - let setting_name = parse_args(env::args()); let mut client = ImdsClient::new(); + let mut aws_k8s_info = api::get_aws_k8s_info().await.context(error::AwsInfoSnafu)?; + + generate_cluster_dns_ip(&mut client, &mut aws_k8s_info).await?; + generate_node_ip(&mut client, &mut aws_k8s_info).await?; + generate_max_pods(&mut client, &mut aws_k8s_info).await?; + generate_provider_id(&mut client, &mut aws_k8s_info).await?; + generate_private_dns_name(&mut client, &mut aws_k8s_info).await?; + + let settings = serde_json::to_value(&aws_k8s_info).context(error::SerializeSnafu)?; + let generated_settings: serde_json::Value = serde_json::json!({ + "kubernetes": settings + }); + let json_str = generated_settings.to_string(); + api::client_command(&["set", "-j", json_str.as_str()]) + .await + .context(error::SetFailureSnafu)?; - let setting = match setting_name.as_ref() { - "cluster-dns-ip" => get_cluster_dns_ip(&mut client).await, - "node-ip" => get_node_ip(&mut client).await, - // If we want to specify a reasonable default in a template, we can exit 2 to tell - // sundog to skip this setting. - "max-pods" => get_max_pods(&mut client) - .await - .map_err(|_| process::exit(2)), - "provider-id" => get_provider_id(&mut client).await, - "private-dns-name" => get_private_dns_name(&mut client).await, - _ => usage(), - }?; - - // sundog expects JSON-serialized output so that many types can be represented, allowing the - // API model to use more accurate types. - - // 'max_pods' setting is an unsigned integer, convert 'settings' to u32 before serializing to JSON - if setting_name == "max-pods" { - let max_pods = serde_json::to_string( - &setting - .parse::() - .context(error::ParseToU32Snafu { setting: &setting })?, - ) - .context(error::OutputJsonSnafu { output: &setting })?; - println!("{}", max_pods); - } else { - let output = - serde_json::to_string(&setting).context(error::OutputJsonSnafu { output: &setting })?; - println!("{}", output); - } Ok(()) } diff --git a/sources/api/pluto/src/proxy.rs b/sources/api/pluto/src/proxy.rs index 446e23ec9b4..a4b10ca2574 100644 --- a/sources/api/pluto/src/proxy.rs +++ b/sources/api/pluto/src/proxy.rs @@ -4,7 +4,6 @@ use hyper::client::HttpConnector; use hyper::Uri; use hyper_rustls::{HttpsConnector, HttpsConnectorBuilder}; use snafu::{ResultExt, Snafu}; -use std::env; use url::Url; #[derive(Debug, Snafu)] @@ -27,21 +26,6 @@ pub(super) enum Error { type Result = std::result::Result; -/// Fetches `HTTPS_PROXY` and `NO_PROXY` variables from the process environment. -pub(crate) fn fetch_proxy_env() -> (Option, Option) { - let https_proxy = ["https_proxy", "HTTPS_PROXY"] - .iter() - .map(env::var) - .find(|env_var| *env_var != Err(env::VarError::NotPresent)) - .and_then(|s| s.ok()); - let no_proxy = ["no_proxy", "NO_PROXY"] - .iter() - .map(env::var) - .find(|env_var| *env_var != Err(env::VarError::NotPresent)) - .and_then(|s| s.ok()); - (https_proxy, no_proxy) -} - /// Setups a hyper-based HTTP client configured with a proxy connector. pub(crate) fn setup_http_client( https_proxy: String, diff --git a/sources/models/shared-defaults/kubernetes-aws-external-cloud-provider.toml b/sources/models/shared-defaults/kubernetes-aws-external-cloud-provider.toml index 0748f1c3929..016b643eaa4 100644 --- a/sources/models/shared-defaults/kubernetes-aws-external-cloud-provider.toml +++ b/sources/models/shared-defaults/kubernetes-aws-external-cloud-provider.toml @@ -1,8 +1,2 @@ [settings.kubernetes] cloud-provider = "external" - -# The host's hostname does not always match the node name expected by the control plane. -# The EC2 instance's 'PrivateDnsName' is what control plane components match against during node registration when -# kubelet advertises the node name. -[metadata.settings.kubernetes] -hostname-override.setting-generator = "pluto private-dns-name" diff --git a/sources/models/shared-defaults/kubernetes-aws.toml b/sources/models/shared-defaults/kubernetes-aws.toml index 9990d92bc83..bbfb625d940 100644 --- a/sources/models/shared-defaults/kubernetes-aws.toml +++ b/sources/models/shared-defaults/kubernetes-aws.toml @@ -6,10 +6,6 @@ server-tls-bootstrap = true cloud-provider = "aws" [metadata.settings.kubernetes] -max-pods.setting-generator = "pluto max-pods" -cluster-dns-ip.setting-generator = "pluto cluster-dns-ip" -node-ip.setting-generator = "pluto node-ip" -provider-id.setting-generator = "pluto provider-id" affected-services = ["kubernetes"] [metadata.settings.kubernetes.pod-infra-container-image] From 5ac2e300dc0a95c26b08a63c4ddec775c54ef867 Mon Sep 17 00:00:00 2001 From: Jarrett Tierney Date: Mon, 6 May 2024 16:32:34 -0700 Subject: [PATCH 3/3] pluto: migration to remove metadata --- Release.toml | 4 ++- sources/Cargo.lock | 7 ++++ sources/Cargo.toml | 1 + .../pluto-remove-generators-v0-1-0/Cargo.toml | 11 +++++++ .../src/main.rs | 33 +++++++++++++++++++ 5 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 sources/api/migration/migrations/v1.21.0/pluto-remove-generators-v0-1-0/Cargo.toml create mode 100644 sources/api/migration/migrations/v1.21.0/pluto-remove-generators-v0-1-0/src/main.rs diff --git a/Release.toml b/Release.toml index 0c456a81830..4a5584e9628 100644 --- a/Release.toml +++ b/Release.toml @@ -312,4 +312,6 @@ version = "1.21.0" "migrate_v1.20.0_aws-control-container-v0-7-12.lz4", "migrate_v1.20.0_public-control-container-v0-7-12.lz4", ] -"(1.20.0, 1.21.0)" = [] +"(1.20.0, 1.21.0)" = [ + "migrate_v1.21.0_pluto-remove-generators-v0-1-0.lz4", +] diff --git a/sources/Cargo.lock b/sources/Cargo.lock index df531f2314d..af5a73d6aba 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -3238,6 +3238,13 @@ dependencies = [ "url", ] +[[package]] +name = "pluto-remove-generators-v0-1-0" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + [[package]] name = "powerfmt" version = "0.2.0" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index dc9c72d0b09..2bc48619630 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -90,6 +90,7 @@ members = [ "api/migration/migrations/v1.20.0/public-admin-container-v0-11-8", "api/migration/migrations/v1.20.0/aws-control-container-v0-7-12", "api/migration/migrations/v1.20.0/public-control-container-v0-7-12", + "api/migration/migrations/v1.21.0/pluto-remove-generators-v0-1-0", "bloodhound", diff --git a/sources/api/migration/migrations/v1.21.0/pluto-remove-generators-v0-1-0/Cargo.toml b/sources/api/migration/migrations/v1.21.0/pluto-remove-generators-v0-1-0/Cargo.toml new file mode 100644 index 00000000000..0a009ea9fa5 --- /dev/null +++ b/sources/api/migration/migrations/v1.21.0/pluto-remove-generators-v0-1-0/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "pluto-remove-generators-v0-1-0" +version = "0.1.0" +authors = ["Jarrett Tierney "] +license = "Apache-2.0 OR MIT" +edition = "2021" +publish = false +exclude = ["README.md"] + +[dependencies] +migration-helpers = { path = "../../../migration-helpers", version = "0.1.0" } diff --git a/sources/api/migration/migrations/v1.21.0/pluto-remove-generators-v0-1-0/src/main.rs b/sources/api/migration/migrations/v1.21.0/pluto-remove-generators-v0-1-0/src/main.rs new file mode 100644 index 00000000000..3b86c5d24a7 --- /dev/null +++ b/sources/api/migration/migrations/v1.21.0/pluto-remove-generators-v0-1-0/src/main.rs @@ -0,0 +1,33 @@ +use migration_helpers::common_migrations::{RemoveMetadataMigration, SettingMetadata}; +use migration_helpers::{migrate, Result}; +use std::process; +fn run() -> Result<()> { + migrate(RemoveMetadataMigration(&[ + SettingMetadata { + setting: "settings.kubernetes.max-pods", + metadata: &["setting-generator"], + }, + SettingMetadata { + setting: "settings.kubernetes.cluster-dns-ip", + metadata: &["setting-generator"], + }, + SettingMetadata { + setting: "settings.kubernetes.node-ip", + metadata: &["setting-generator"], + }, + SettingMetadata { + setting: "settings.kubernetes.provider-id", + metadata: &["setting-generator"], + }, + SettingMetadata { + setting: "settings.kubernetes.hostname-override", + metadata: &["setting-generator"], + }, + ])) +} +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +}