chore: bump operator rs 0.67 (#614)

* WIP

* WIP

* WIP

* WIP

* Update Cargo.nix

* fix built

* clippy fix

* review fix - default

* review fix - update op-rs

* use full qualified name for Error

* regenerate-nix

* Update clap

* regenerate charts

Author: fhennig

Cargo.lock

@@ -11,8 +11,8 @@ repository = "https://github.com/stackabletech/nifi-operator"
 
 [workspace.dependencies]
 anyhow = "1.0"
-built = { version = "0.6", features = ["chrono", "git2"] }
-clap = "4.3"
+built = { version = "0.7", features = ["chrono", "git2"] }
+clap = "4.5"
 fnv = "1.0"
 futures = { version = "0.3", features = ["compat"] }
 indoc = "2.0"
@@ -23,10 +23,10 @@ semver = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
-snafu = "0.7"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.64.0" }
-strum = { version = "0.25", features = ["derive"] }
-tokio = { version = "1.29", features = ["full"] }
+snafu = "0.8"
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.67.1" }
+strum = { version = "0.26", features = ["derive"] }
+tokio = { version = "1.37", features = ["full"] }
 tracing = "0.1"
 xml-rs = "0.8"
 

Cargo.nix

@@ -1,8 +1,11 @@
 = Usage guide
 
-This section will help you to use various aspects of the Stackable Operator for Apache NiFi. For a general introduction into the operator follow the xref:getting_started/index.adoc[] guide. Below is a general overview of some configuration aspects, have a look at the subpages for details.
+This section will help you to use various aspects of the Stackable Operator for Apache NiFi.
+For a general introduction into the operator follow the xref:getting_started/index.adoc[] guide.
+Below is a general overview of some configuration aspects, have a look at the subpages for details.
 
-The cluster is configured via a YAML manifest file. This custom resource specifies the amount of replicas for each role group or role specific configuration like resource requests.
+The cluster is configured via a YAML manifest file.
+This custom resource specifies the amount of replicas for each role group or role specific configuration like resource requests.
 The following listing shows an example configuration:
 
 [source,yaml]

Cargo.toml

@@ -1,8 +1,11 @@
 = Service exposition with ListenerClasses
 
-Apache NiFi offers a web UI and an API. The Operator deploys a service called `<name>` (where `<name>` is the name of the NifiCluster) through which NiFi can be reached.
+Apache NiFi offers a web UI and an API.
+The Operator deploys a service called `<name>` (where `<name>` is the name of the NifiCluster) through which NiFi can be reached.
 
-This service can have either the `cluster-internal` or `external-unstable` type. `external-stable` is not supported for NiFi at the moment. Read more about the types in the xref:concepts:service-exposition.adoc[service exposition] documentation at platform level.
+This service can have either the `cluster-internal` or `external-unstable` type.
+`external-stable` is not supported for NiFi at the moment.
+Read more about the types in the xref:concepts:service-exposition.adoc[service exposition] documentation at platform level.
 
 This is how the listener class is configured:
 

crate-hashes.json

@@ -1,7 +1,6 @@
 = Log aggregation
 
-The logs can be forwarded to a Vector log aggregator by providing a discovery
-ConfigMap for the aggregator and by enabling the log agent:
+The logs can be forwarded to a Vector log aggregator by providing a discovery ConfigMap for the aggregator and by enabling the log agent:
 
 [source,yaml]
 ----
@@ -14,5 +13,4 @@ spec:
         enableVectorAgent: true
 ----
 
-Further information on how to configure logging, can be found in
-xref:concepts:logging.adoc[].
+Further information on how to configure logging, can be found in xref:concepts:logging.adoc[].

deploy/helm/nifi-operator/crds/crds.yaml

@@ -1,6 +1,7 @@
 = Writing to Iceberg tables
 
-https://iceberg.apache.org/[Apache Iceberg] is a high-performance format for huge analytic tables. Iceberg brings the reliability and simplicity of SQL tables to big data, while making it possible for engines like Spark, Trino, Flink, Presto, Hive and Impala to safely work with the same tables, at the same time.
+https://iceberg.apache.org/[Apache Iceberg] is a high-performance format for huge analytic tables.
+Iceberg brings the reliability and simplicity of SQL tables to big data, while making it possible for engines like Spark, Trino, Flink, Presto, Hive and Impala to safely work with the same tables, at the same time.
 
 NiFi supports a `PutIceberg` processor to add rows to an existing Iceberg table https://issues.apache.org/jira/browse/NIFI-10442[starting from version 1.19.0].
 As of NiFi version `1.23.1` only `PutIceberg` is supported, you need to create and compact your tables with other tools such as Trino or Spark (both included in the SDP).

docs/modules/nifi/pages/usage_guide/index.adoc

@@ -83,9 +83,8 @@ mod tests {
                                         )
                                     ]))
                                 }),
-                                namespace_selector: None,
-                                namespaces: None,
                                 topology_key: "kubernetes.io/hostname".to_string(),
+                                ..Default::default()
                             },
                             weight: 70
                         }

docs/modules/nifi/pages/usage_guide/listenerclass.adoc

@@ -13,7 +13,7 @@ use stackable_operator::{
 pub enum Error {
     #[snafu(display("Failed to retrieve AuthenticationClass {authentication_class}"))]
     AuthenticationClassRetrieval {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::client::Error,
         authentication_class: ObjectRef<AuthenticationClass>,
     },
     #[snafu(display("The nifi-operator does not support running Nifi without any authentication. Please provide a AuthenticationClass to use."))]

docs/modules/nifi/pages/usage_guide/log-aggregation.adoc

@@ -24,7 +24,7 @@ use stackable_operator::{
     k8s_openapi::{api::core::v1::Volume, apimachinery::pkg::api::resource::Quantity},
     kube::{runtime::reflector::ObjectRef, CustomResource, ResourceExt},
     memory::{BinaryMultiple, MemoryQuantity},
-    product_config_utils::{ConfigError, Configuration},
+    product_config_utils::{self, Configuration},
     product_logging::{self, spec::Logging},
     role_utils::{GenericRoleConfig, Role, RoleGroupRef},
     schemars::{self, JsonSchema},
@@ -382,15 +382,15 @@ impl Configuration for NifiConfigFragment {
         &self,
         _resource: &Self::Configurable,
         _role_name: &str,
-    ) -> Result<BTreeMap<String, Option<String>>, ConfigError> {
+    ) -> Result<BTreeMap<String, Option<String>>, product_config_utils::Error> {
         Ok(BTreeMap::new())
     }
 
     fn compute_cli(
         &self,
         _resource: &Self::Configurable,
         _role_name: &str,
-    ) -> Result<BTreeMap<String, Option<String>>, ConfigError> {
+    ) -> Result<BTreeMap<String, Option<String>>, product_config_utils::Error> {
         Ok(BTreeMap::new())
     }
 
@@ -399,7 +399,7 @@ impl Configuration for NifiConfigFragment {
         _resource: &Self::Configurable,
         _role_name: &str,
         _file: &str,
-    ) -> Result<BTreeMap<String, Option<String>>, ConfigError> {
+    ) -> Result<BTreeMap<String, Option<String>>, product_config_utils::Error> {
         Ok(BTreeMap::new())
     }
 }

docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc

@@ -1,13 +1,3 @@
-use std::path::PathBuf;
-
 fn main() {
-    let out_dir = PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR is required"));
-    built::write_built_file_with_opts(
-        // built's env module depends on a whole bunch of variables that crate2nix doesn't provide
-        // so we grab the specific env variables that we care about out ourselves instead.
-        built::Options::default().set_env(false),
-        "Cargo.toml".as_ref(),
-        &out_dir.join("built.rs"),
-    )
-    .unwrap();
+    built::write_built_file().unwrap();
 }

rust/crd/src/affinity.rs

@@ -67,15 +67,19 @@ impl NifiRepository {
 pub enum Error {
     #[snafu(display("Invalid product config"))]
     InvalidProductConfig {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::product_config_utils::Error,
+    },
+    #[snafu(display("Invalid memory config"))]
+    InvalidMemoryConfig {
+        source: stackable_operator::memory::Error,
     },
     #[snafu(display("Failed to transform product configs"))]
     ProductConfigTransform {
-        source: stackable_operator::product_config_utils::ConfigError,
+        source: stackable_operator::product_config_utils::Error,
     },
     #[snafu(display("failed to calculate storage quota for {repo} repository"))]
     CalculateStorageQuota {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::memory::Error,
         repo: NifiRepository,
     },
 }
@@ -106,13 +110,13 @@ pub fn build_bootstrap_conf(
         tracing::debug!("Read {:?} from crd as memory limit", heap_size_definition);
 
         let heap_size = MemoryQuantity::try_from(heap_size_definition)
-            .context(InvalidProductConfigSnafu)?
+            .context(InvalidMemoryConfigSnafu)?
             .scale_to(BinaryMultiple::Mebi)
             * JAVA_HEAP_FACTOR;
 
         let java_heap = heap_size
             .format_for_java()
-            .context(InvalidProductConfigSnafu)?;
+            .context(InvalidMemoryConfigSnafu)?;
 
         tracing::debug!(
             "Converted {:?} to {} for java heap config",

rust/crd/src/authentication.rs

@@ -22,8 +22,12 @@ use stackable_nifi_crd::{
 };
 use stackable_operator::{
     builder::{
-        resources::ResourceRequirementsBuilder, ConfigMapBuilder, ContainerBuilder,
-        ObjectMetaBuilder, PodBuilder, PodSecurityContextBuilder, SecretFormat,
+        configmap::ConfigMapBuilder,
+        meta::ObjectMetaBuilder,
+        pod::{
+            container::ContainerBuilder, resources::ResourceRequirementsBuilder,
+            security::PodSecurityContextBuilder, volume::SecretFormat, PodBuilder,
+        },
     },
     client::Client,
     cluster_resources::{ClusterResourceApplyStrategy, ClusterResources},
@@ -113,33 +117,38 @@ pub enum Error {
 
     #[snafu(display("failed to create cluster resources"))]
     CreateClusterResources {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::cluster_resources::Error,
     },
 
     #[snafu(display("failed to delete orphaned resources"))]
     DeleteOrphanedResources {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::cluster_resources::Error,
     },
 
     #[snafu(display("failed to apply global Service"))]
     ApplyRoleService {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::cluster_resources::Error,
+    },
+
+    #[snafu(display("failed to fetch deployed StatefulSets"))]
+    FetchStatefulsets {
+        source: stackable_operator::client::Error,
     },
 
     #[snafu(display("failed to update status"))]
     StatusUpdate {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::client::Error,
     },
 
     #[snafu(display("failed to apply Service for {}", rolegroup))]
     ApplyRoleGroupService {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::cluster_resources::Error,
         rolegroup: RoleGroupRef<NifiCluster>,
     },
 
     #[snafu(display("failed to build ConfigMap for {}", rolegroup))]
     BuildRoleGroupConfig {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::builder::configmap::Error,
         rolegroup: RoleGroupRef<NifiCluster>,
     },
 
@@ -148,46 +157,49 @@ pub enum Error {
 
     #[snafu(display("failed to apply ConfigMap for {}", rolegroup))]
     ApplyRoleGroupConfig {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::cluster_resources::Error,
         rolegroup: RoleGroupRef<NifiCluster>,
     },
 
     #[snafu(display("failed to apply StatefulSet for {}", rolegroup))]
     ApplyRoleGroupStatefulSet {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::cluster_resources::Error,
         rolegroup: RoleGroupRef<NifiCluster>,
     },
 
     #[snafu(display("failed to apply create ReportingTask service"))]
     ApplyCreateReportingTaskService {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::cluster_resources::Error,
     },
 
     #[snafu(display("failed to apply create ReportingTask job"))]
     ApplyCreateReportingTaskJob {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::cluster_resources::Error,
     },
 
     #[snafu(display("object is missing metadata to build owner reference"))]
     ObjectMissingMetadataForOwnerRef {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::builder::meta::Error,
     },
 
-    #[snafu(display("Failed to load Product Config"))]
-    ProductConfigLoadFailed { source: config::Error },
+    #[snafu(display("Failed to load product config"))]
+    ProductConfigLoadFailed {
+        #[snafu(source(from(config::Error, Box::new)))]
+        source: Box<config::Error>,
+    },
 
     #[snafu(display("Failed to find information about file [{}] in product config", kind))]
     ProductConfigKindNotSpecified { kind: String },
 
     #[snafu(display("Failed to find any nodes in cluster {obj_ref}",))]
     MissingNodes {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::client::Error,
         obj_ref: ObjectRef<NifiCluster>,
     },
 
     #[snafu(display("Failed to find service {obj_ref}"))]
     MissingService {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::client::Error,
         obj_ref: ObjectRef<Service>,
     },
 
@@ -198,17 +210,21 @@ pub enum Error {
     NoRoleServiceFqdn,
 
     #[snafu(display("Bootstrap configuration error"))]
-    BoostrapConfig { source: crate::config::Error },
+    BootstrapConfig {
+        #[snafu(source(from(config::Error, Box::new)))]
+        source: Box<config::Error>,
+    },
 
     #[snafu(display("failed to prepare NiFi configuration for rolegroup {rolegroup}"))]
     BuildProductConfig {
-        source: crate::config::Error,
+        #[snafu(source(from(config::Error, Box::new)))]
+        source: Box<config::Error>,
         rolegroup: RoleGroupRef<NifiCluster>,
     },
 
     #[snafu(display("illegal container name: [{container_name}]"))]
     IllegalContainerName {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::builder::pod::container::Error,
         container_name: String,
     },
 
@@ -234,17 +250,17 @@ pub enum Error {
 
     #[snafu(display("failed to patch service account"))]
     ApplyServiceAccount {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::cluster_resources::Error,
     },
 
     #[snafu(display("failed to patch role binding"))]
     ApplyRoleBinding {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::cluster_resources::Error,
     },
 
     #[snafu(display("failed to build RBAC resources"))]
     BuildRbacResources {
-        source: stackable_operator::error::Error,
+        source: stackable_operator::commons::rbac::Error,
     },
 
     #[snafu(display(
@@ -278,7 +294,7 @@ pub enum Error {
 
     #[snafu(display("failed to build metadata"))]
     MetadataBuild {
-        source: stackable_operator::builder::ObjectMetaBuilderError,
+        source: stackable_operator::builder::meta::Error,
     },
 
     #[snafu(display("failed to get required labels"))]
@@ -333,7 +349,7 @@ pub async fn reconcile_nifi(nifi: Arc<NifiCluster>, ctx: Arc<Ctx>) -> Result<Act
     let resolved_product_image: ResolvedProductImage = nifi
         .spec
         .image
-        .resolve(DOCKER_IMAGE_BASE_NAME, crate::built_info::CARGO_PKG_VERSION);
+        .resolve(DOCKER_IMAGE_BASE_NAME, crate::built_info::PKG_VERSION);
 
     tracing::info!("Checking for sensitive key configuration");
     check_or_generate_sensitive_key(client, &nifi)
@@ -361,7 +377,7 @@ pub async fn reconcile_nifi(nifi: Arc<NifiCluster>, ctx: Arc<Ctx>) -> Result<Act
             let deployed_statefulsets = client
                 .list_with_label_selector::<StatefulSet>(namespace, &selector)
                 .await
-                .context(ApplyRoleServiceSnafu)?;
+                .context(FetchStatefulsetsSnafu)?;
 
             // Sum target replicas for all statefulsets
             let target_replicas = deployed_statefulsets
@@ -722,7 +738,7 @@ async fn build_node_rolegroup_config_map(
                     })?
                     .clone(),
             )
-            .context(BoostrapConfigSnafu)?,
+            .context(BootstrapConfigSnafu)?,
         )
         .add_data(
             NIFI_PROPERTIES,