diff --git a/nexus/db-model/src/external_service_ip_pool.rs b/nexus/db-model/src/external_service_ip_pool.rs new file mode 100644 index 00000000000..9217b8d3d50 --- /dev/null +++ b/nexus/db-model/src/external_service_ip_pool.rs @@ -0,0 +1,65 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Model types for assigning IP pools to individual external services. + +use super::impl_enum_type; +use crate::DbTypedUuid; +use diesel::Insertable; +use diesel::Queryable; +use diesel::Selectable; +use nexus_db_schema::schema::external_service_ip_pool; +use omicron_uuid_kinds::IpPoolKind; +use serde::Deserialize; +use serde::Serialize; + +impl_enum_type!( + ExternalServiceKindEnum: + + #[derive( + Copy, + Clone, + Debug, + PartialEq, + Eq, + Serialize, + Deserialize, + AsExpression, + FromSqlRow, + )] + #[serde(rename_all = "snake_case")] + pub enum ExternalServiceKind; + + Nexus => b"nexus" + BoundaryNtp => b"boundary_ntp" + ExternalDns => b"external_dns" +); + +impl std::fmt::Display for ExternalServiceKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + ExternalServiceKind::Nexus => "nexus", + ExternalServiceKind::BoundaryNtp => "boundary_ntp", + ExternalServiceKind::ExternalDns => "external_dns", + }; + f.write_str(s) + } +} + +/// Assignment of an IP pool to an external service. +// +// NOTE: The associations here intentionally avoid any semantics like +// exclusivity or what an assignment means for the application. Each service +// could be different in that respect, and we haven't really fleshed out enough +// of the system to know how to use the records here. For now, we just enforce +// basic sanity checks, like that operators cannot delete the last pool assigned +// to a service, or concurrent operations on the pool and this record. +#[derive( + Queryable, Clone, Copy, Debug, PartialEq, Eq, Selectable, Insertable, +)] +#[diesel(table_name = external_service_ip_pool)] +pub struct ExternalServiceIpPool { + pub service: ExternalServiceKind, + pub ip_pool_id: DbTypedUuid, +} diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index 34a93f69f7f..eb3fbebea97 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -151,6 +151,7 @@ pub mod ereport; mod ereporter_restart; mod ereporter_type; mod external_ip; +mod external_service_ip_pool; mod external_subnet; pub mod fm; mod generation; @@ -306,6 +307,7 @@ pub use ereport::Ereport; pub use ereporter_restart::*; pub use ereporter_type::*; pub use external_ip::*; +pub use external_service_ip_pool::*; pub use external_subnet::*; pub use fm::{SitrepMetadata, SitrepVersion}; pub use generation::*; diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index c7222b1327e..b12927957d3 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock}; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: Version = Version::new(293, 0, 0); +pub const SCHEMA_VERSION: Version = Version::new(294, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -28,6 +28,7 @@ pub static KNOWN_VERSIONS: LazyLock> = LazyLock::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(294, "external-service-ip-pool"), KnownVersion::new(293, "add-rendezvous-sled-bp-availability"), KnownVersion::new(292, "allow-ddm-traffic"), KnownVersion::new(291, "bgp-peer-src-addr"), diff --git a/nexus/db-queries/src/db/datastore/external_service_ip_pool.rs b/nexus/db-queries/src/db/datastore/external_service_ip_pool.rs new file mode 100644 index 00000000000..29b4859ba21 --- /dev/null +++ b/nexus/db-queries/src/db/datastore/external_service_ip_pool.rs @@ -0,0 +1,891 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! [`DataStore`] methods for assigning IP pools to external services. +//! +//! These are the internal persistence methods behind the (not-yet-built) +//! per-service networking configuration API. The assignment is operator intent +//! consumed by Reconfigurator, but nothing reads this table yet. + +use crate::authz; +use crate::context::OpContext; +use crate::db::datastore::DataStore; +use crate::db::pagination::paginated; +use crate::db::pagination::paginated_multicolumn; +use crate::db::raw_query_builder::QueryBuilder; +use crate::db::raw_query_builder::TypedSqlQuery; +use crate::db::true_or_cast_error::matches_sentinel; +use async_bb8_diesel::AsyncRunQueryDsl; +use diesel::prelude::*; +use diesel::result::DatabaseErrorKind; +use diesel::result::Error as DieselError; +use diesel::sql_types; +use nexus_db_errors::ErrorHandler; +use nexus_db_errors::public_error_from_diesel; +use nexus_db_model::ExternalServiceIpPool; +use nexus_db_model::ExternalServiceKind; +use nexus_db_model::IpPoolAssignment; +use nexus_db_schema::enums::ExternalServiceKindEnum; +use nexus_db_schema::enums::IpPoolAssignmentEnum; +use omicron_common::api::external::DataPageParams; +use omicron_common::api::external::Error; +use omicron_common::api::external::ListResultVec; +use omicron_common::api::external::ResourceType; +use uuid::Uuid; + +// Bool-cast sentinel used when a target IP Pool exists, but isn't assigned for +// `system_services` at all. +const NOT_SYSTEM_SERVICES_POOL_SENTINEL: &str = "not-a-system-services-pool"; + +// Bool-cast sentinel used when we try to unassign the last pool for a specific +// service. +const UNASSIGN_LAST_POOL_SENTINEL: &str = "unassign-last-pool"; +const UNASSIGN_LAST_POOL_ERROR: &str = + "cannot remove the last IP pool assigned to a service"; + +impl DataStore { + /// Assign an IP pool to an external service. + /// + /// This fails if the IP Pool doesn't exist or exists but isn't a service IP + /// Pool. Assigning the same pool multiple times is a no-op. + pub async fn external_service_ip_pool_assign( + &self, + opctx: &OpContext, + service: ExternalServiceKind, + authz_pool: &authz::IpPool, + ) -> Result<(), Error> { + opctx.authorize(authz::Action::Modify, authz_pool).await?; + let pool_id = authz_pool.id(); + let conn = self.pool_connection_authorized(opctx).await?; + + match assign_pool_to_service_query(service, pool_id) + .execute_async(&*conn) + .await + { + // No rows means there is no such pool at all. + Ok(0) => { + Err(Error::not_found_by_id(ResourceType::IpPool, &pool_id)) + } + Ok(_) => Ok(()), + // Already exists -> no-op. + Err(DieselError::DatabaseError( + DatabaseErrorKind::UniqueViolation, + _, + )) => Ok(()), + // Pool exists, but is a silo pool. + Err(e) + if matches_sentinel( + &e, + &[NOT_SYSTEM_SERVICES_POOL_SENTINEL], + ) + .is_some() => + { + Err(Error::invalid_request( + "only IP pools assigned for system services may be \ + assigned to a specific service", + )) + } + Err(e) => Err(public_error_from_diesel(e, ErrorHandler::Server)), + } + } + + /// Unassign an IP pool from an external service. + /// + /// This fails if the pool is the last one assigned to the specific service. + /// Unassigning a pool multiple times is a no-op. + pub async fn external_service_ip_pool_unassign( + &self, + opctx: &OpContext, + service: ExternalServiceKind, + authz_pool: &authz::IpPool, + ) -> Result<(), Error> { + opctx.authorize(authz::Action::Modify, authz_pool).await?; + let pool_id = authz_pool.id(); + let conn = self.pool_connection_authorized(opctx).await?; + + match unassign_pool_from_service_query(service, pool_id) + .execute_async(&*conn) + .await + { + // A row was removed, or the pair wasn't assigned; either is success. + Ok(_) => Ok(()), + // The bool-cast sentinel fired: this was the service's last pool. + Err(e) + if matches_sentinel(&e, &[UNASSIGN_LAST_POOL_SENTINEL]) + .is_some() => + { + Err(Error::invalid_request(UNASSIGN_LAST_POOL_ERROR)) + } + Err(e) => Err(public_error_from_diesel(e, ErrorHandler::Server)), + } + } + + /// List assignments between pools and services. + pub async fn external_service_ip_pool_list( + &self, + opctx: &OpContext, + pagparams: &DataPageParams<'_, (ExternalServiceKind, Uuid)>, + ) -> ListResultVec { + use nexus_db_schema::schema::external_service_ip_pool; + opctx + .authorize(authz::Action::ListChildren, &authz::IP_POOL_LIST) + .await?; + let conn = self.pool_connection_authorized(opctx).await?; + paginated_multicolumn( + external_service_ip_pool::table, + ( + external_service_ip_pool::service, + external_service_ip_pool::ip_pool_id, + ), + pagparams, + ) + .select(ExternalServiceIpPool::as_select()) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + /// List IP pools assigned to a given external service. + pub async fn external_service_ip_pool_pools_for_service( + &self, + opctx: &OpContext, + service: ExternalServiceKind, + pagparams: &DataPageParams<'_, Uuid>, + ) -> ListResultVec { + use nexus_db_schema::schema::external_service_ip_pool; + opctx + .authorize(authz::Action::ListChildren, &authz::IP_POOL_LIST) + .await?; + let conn = self.pool_connection_authorized(opctx).await?; + paginated( + external_service_ip_pool::table, + external_service_ip_pool::ip_pool_id, + pagparams, + ) + .filter(external_service_ip_pool::service.eq(service)) + .select(external_service_ip_pool::ip_pool_id) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } +} + +// Query to assign a pool to a service. +// +// This emits a custom query which: +// +// - Checks that the pool exists, isn't deleted, and is a service IP Pool. +// - Increments the `ip_pool.rcgen` column, to detect concurrent changes to the +// pool and this assignment. +// +// The return value helps disambiguate the failure modes: +// +// - If the pool doesn't exist, Ok(0) is returned (no rows affected) +// - If the pool isn't a service pool, a bool-cast error is generated +// - If the assignment already existed, a PK violation is generated. +// - Otherwise, the assignment succeeds and the row count is returned. +fn assign_pool_to_service_query( + service: ExternalServiceKind, + pool_id: Uuid, +) -> TypedSqlQuery<()> { + let mut builder = QueryBuilder::new(); + builder + .sql("WITH bumped AS (UPDATE ip_pool SET rcgen = rcgen + 1 WHERE id = ") + .param() + .bind::(pool_id) + .sql(" AND time_deleted IS NULL AND CAST(IF(assignment = ") + .param() + .bind::(IpPoolAssignment::SystemServices) + .sql(", 'true', '") + .sql(NOT_SYSTEM_SERVICES_POOL_SENTINEL) + .sql( + "') AS BOOL) RETURNING id) \ + INSERT INTO external_service_ip_pool (service, ip_pool_id) SELECT ", + ) + .param() + .bind::(service) + .sql(", id FROM bumped"); + builder.query() +} + +// Query to unassign a pool from a service. +// +// This generates a query which: +// +// - Hard-deletes the pool <-> service assignment +// - Increments the `ip_pool.rcgen` generation number to detect concurrent +// operations on the pool and this association. +// +// The return values disambiguates the failure modes: +// +// - If there isn't at least 1 other pool assigned to the same service, a +// bool-cast error is generated. +// - If the pool isn't currently assigned, Ok(0) is returned (no rows affected). +// - Otherwise, the unassignment succeeds and the row count is returned. +fn unassign_pool_from_service_query( + service: ExternalServiceKind, + pool_id: Uuid, +) -> TypedSqlQuery<()> { + let mut builder = QueryBuilder::new(); + builder + .sql( + "WITH deleted AS (\ + DELETE FROM external_service_ip_pool WHERE service = ", + ) + .param() + .bind::(service) + .sql(" AND ip_pool_id = ") + .param() + .bind::(pool_id) + .sql( + " AND CAST(IF(\ + (SELECT COUNT(1) FROM external_service_ip_pool WHERE service = ", + ) + .param() + .bind::(service) + .sql(") >= 2, 'true', '") + .sql(UNASSIGN_LAST_POOL_SENTINEL) + .sql( + "') AS BOOL) RETURNING ip_pool_id) \ + UPDATE ip_pool SET rcgen = rcgen + 1 WHERE id = ", + ) + .param() + .bind::(pool_id) + .sql(" AND EXISTS (SELECT 1 FROM deleted)"); + builder.query() +} + +#[cfg(test)] +mod test { + use super::assign_pool_to_service_query; + use super::unassign_pool_from_service_query; + use crate::authz; + use crate::db::datastore::DataStore; + use crate::db::pagination::Paginator; + use crate::db::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::helpers::create_service_ip_pool; + use crate::db::raw_query_builder::expectorate_query_contents; + use assert_matches::assert_matches; + use async_bb8_diesel::AsyncRunQueryDsl as _; + use diesel::ExpressionMethods as _; + use diesel::QueryDsl as _; + use nexus_db_model::ExternalServiceIpPool; + use nexus_db_model::ExternalServiceKind; + use nexus_db_model::IpPool; + use nexus_db_model::IpPoolAssignment; + use nexus_types::identity::Resource as _; + use omicron_common::api::external::DataPageParams; + use omicron_common::api::external::Error; + use omicron_common::api::external::IdentityMetadataCreateParams; + use omicron_common::api::external::IpVersion; + use omicron_common::api::external::LookupType; + use omicron_test_utils::dev; + use omicron_uuid_kinds::GenericUuid as _; + use uuid::Uuid; + + // Read a pool's current `rcgen`, to check that mutations bump it. + async fn pool_rcgen(datastore: &DataStore, pool_id: Uuid) -> i64 { + use nexus_db_schema::schema::ip_pool::dsl; + let conn = datastore + .pool_connection_for_tests() + .await + .expect("got a connection"); + dsl::ip_pool + .filter(dsl::id.eq(pool_id)) + .select(dsl::rcgen) + .first_async::(&*conn) + .await + .expect("read the pool's rcgen") + } + + #[tokio::test] + async fn test_assign_pool_to_service() { + let logctx = dev::test_setup_log("test_assign_pool_to_service"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // A system-services IP pool, and its `rcgen` before we touch it. + let pool = create_service_ip_pool( + opctx, + datastore, + "svc-pool-v4", + IpVersion::V4, + ) + .await; + let pool_id = pool.authz_pool.id(); + let rcgen_before = pool_rcgen(datastore, pool_id).await; + + // Assign it to Nexus. + datastore + .external_service_ip_pool_assign( + opctx, + ExternalServiceKind::Nexus, + &pool.authz_pool, + ) + .await + .expect("assigning a system-services pool to Nexus should succeed"); + + // Shows up in the Nexus-specific list, and the pool's `rcgen` was + // bumped. + let pools = datastore + .external_service_ip_pool_pools_for_service( + opctx, + ExternalServiceKind::Nexus, + &DataPageParams::max_page(), + ) + .await + .expect("listing Nexus's pools should succeed"); + assert_eq!(pools, vec![pool_id]); + assert_eq!(pool_rcgen(datastore, pool_id).await, rcgen_before + 1); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_assign_is_idempotent() { + let logctx = dev::test_setup_log("test_assign_is_idempotent"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let pool = create_service_ip_pool( + opctx, + datastore, + "svc-pool-v4", + IpVersion::V4, + ) + .await; + let pool_id = pool.authz_pool.id(); + + // First assignment. + datastore + .external_service_ip_pool_assign( + opctx, + ExternalServiceKind::Nexus, + &pool.authz_pool, + ) + .await + .expect("first assignment should succeed"); + let rcgen_after_first = pool_rcgen(datastore, pool_id).await; + + // Re-assigning the same pair is a no-op. + datastore + .external_service_ip_pool_assign( + opctx, + ExternalServiceKind::Nexus, + &pool.authz_pool, + ) + .await + .expect("re-assigning the same pair should succeed"); + + let pools = datastore + .external_service_ip_pool_pools_for_service( + opctx, + ExternalServiceKind::Nexus, + &DataPageParams::max_page(), + ) + .await + .expect("listing Nexus's pools should succeed"); + assert_eq!(pools, vec![pool_id]); + assert_eq!(pool_rcgen(datastore, pool_id).await, rcgen_after_first); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_pool_shared_by_two_services() { + let logctx = dev::test_setup_log("test_pool_shared_by_two_services"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let pool = create_service_ip_pool( + opctx, + datastore, + "svc-pool-v4", + IpVersion::V4, + ) + .await; + let pool_id = pool.authz_pool.id(); + let rcgen_before = pool_rcgen(datastore, pool_id).await; + + // The same pool can back more than one service. + for service in + [ExternalServiceKind::Nexus, ExternalServiceKind::BoundaryNtp] + { + datastore + .external_service_ip_pool_assign( + opctx, + service, + &pool.authz_pool, + ) + .await + .expect("assigning the pool to a service should succeed"); + } + + // Shows up in the list for each service. + for service in + [ExternalServiceKind::Nexus, ExternalServiceKind::BoundaryNtp] + { + let pools = datastore + .external_service_ip_pool_pools_for_service( + opctx, + service, + &DataPageParams::max_page(), + ) + .await + .expect("listing a service's pools should succeed"); + assert_eq!(pools, vec![pool_id]); + } + + // Both rows show up in the full list, for any service. + let all_rows = datastore + .external_service_ip_pool_list(opctx, &DataPageParams::max_page()) + .await + .expect("listing all assignments should succeed"); + let got: Vec<(ExternalServiceKind, Uuid)> = all_rows + .iter() + .map(|r| (r.service, r.ip_pool_id.into_untyped_uuid())) + .collect(); + assert_eq!(got.len(), 2); + assert!(got.contains(&(ExternalServiceKind::Nexus, pool_id))); + assert!(got.contains(&(ExternalServiceKind::BoundaryNtp, pool_id))); + + // Two assignments, two rcgen bumps. + assert_eq!(pool_rcgen(datastore, pool_id).await, rcgen_before + 2); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_assign_silo_pool_is_rejected() { + let logctx = dev::test_setup_log("test_assign_silo_pool_is_rejected"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // A silo pool, not a system-services pool. + let db_pool = datastore + .ip_pool_create( + opctx, + IpPool::new( + &IdentityMetadataCreateParams { + name: "silo-pool".parse().unwrap(), + description: String::new(), + }, + nexus_db_model::IpVersion::V4, + IpPoolAssignment::Silos, + ), + ) + .await + .expect("creating a silo pool should succeed"); + let authz_pool = authz::IpPool::new( + authz::FLEET, + db_pool.id(), + LookupType::ById(db_pool.id()), + ); + + // Assigning a silo pool to a service is a 400. + let err = datastore + .external_service_ip_pool_assign( + opctx, + ExternalServiceKind::Nexus, + &authz_pool, + ) + .await + .expect_err("assigning a silo pool to a service should fail"); + assert_matches!(err, Error::InvalidRequest { .. }); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_assign_nonexistent_pool_is_not_found() { + let logctx = + dev::test_setup_log("test_assign_nonexistent_pool_is_not_found"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // An authz handle for a pool that was never created. + let missing = uuid::uuid!("289b779c-f8ec-4b04-97fd-39e70905d924"); + let authz_pool = authz::IpPool::new( + authz::FLEET, + missing, + LookupType::ById(missing), + ); + + let err = datastore + .external_service_ip_pool_assign( + opctx, + ExternalServiceKind::Nexus, + &authz_pool, + ) + .await + .expect_err("assigning a non-existent pool should fail"); + assert_matches!(err, Error::ObjectNotFound { .. }); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_unassign_pool() { + let logctx = dev::test_setup_log("test_unassign_pool"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let pool1 = create_service_ip_pool( + opctx, + datastore, + "svc-pool-1", + IpVersion::V4, + ) + .await; + let pool2 = create_service_ip_pool( + opctx, + datastore, + "svc-pool-2", + IpVersion::V4, + ) + .await; + let id1 = pool1.authz_pool.id(); + let id2 = pool2.authz_pool.id(); + + // Nexus gets both pools. + for pool in [&pool1, &pool2] { + datastore + .external_service_ip_pool_assign( + opctx, + ExternalServiceKind::Nexus, + &pool.authz_pool, + ) + .await + .expect("assigning should succeed"); + } + let rcgen1_before = pool_rcgen(datastore, id1).await; + + // Removing pool1 succeeds because pool2 still remains. + datastore + .external_service_ip_pool_unassign( + opctx, + ExternalServiceKind::Nexus, + &pool1.authz_pool, + ) + .await + .expect("unassigning should succeed while another pool remains"); + + // Only pool2 is left, and pool1's rcgen was bumped by the removal. + let pools = datastore + .external_service_ip_pool_pools_for_service( + opctx, + ExternalServiceKind::Nexus, + &DataPageParams::max_page(), + ) + .await + .expect("listing Nexus's pools should succeed"); + assert_eq!(pools, vec![id2]); + assert_eq!(pool_rcgen(datastore, id1).await, rcgen1_before + 1); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_unassign_last_pool_is_rejected() { + let logctx = dev::test_setup_log("test_unassign_last_pool_is_rejected"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let pool = create_service_ip_pool( + opctx, + datastore, + "svc-pool-v4", + IpVersion::V4, + ) + .await; + let pool_id = pool.authz_pool.id(); + datastore + .external_service_ip_pool_assign( + opctx, + ExternalServiceKind::Nexus, + &pool.authz_pool, + ) + .await + .expect("assigning should succeed"); + let rcgen_before = pool_rcgen(datastore, pool_id).await; + + // It's Nexus's only pool, so removing it is rejected as a 400. + let err = datastore + .external_service_ip_pool_unassign( + opctx, + ExternalServiceKind::Nexus, + &pool.authz_pool, + ) + .await + .expect_err("removing a service's last pool should fail"); + assert_matches!(err, Error::InvalidRequest { .. }); + + // Still assigned, and rcgen not changed. + let pools = datastore + .external_service_ip_pool_pools_for_service( + opctx, + ExternalServiceKind::Nexus, + &DataPageParams::max_page(), + ) + .await + .expect("listing Nexus's pools should succeed"); + assert_eq!(pools, vec![pool_id]); + assert_eq!(pool_rcgen(datastore, pool_id).await, rcgen_before); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_unassign_not_assigned_is_noop() { + let logctx = dev::test_setup_log("test_unassign_not_assigned_is_noop"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let pool = create_service_ip_pool( + opctx, + datastore, + "svc-pool-v4", + IpVersion::V4, + ) + .await; + let pool_id = pool.authz_pool.id(); + let rcgen_before = pool_rcgen(datastore, pool_id).await; + + // The pool was never assigned to Nexus, removing is a no-op. + datastore + .external_service_ip_pool_unassign( + opctx, + ExternalServiceKind::Nexus, + &pool.authz_pool, + ) + .await + .expect("unassigning a not-assigned pair should be a no-op"); + + // Still doesn't show up in the list for Nexus. + let pools = datastore + .external_service_ip_pool_pools_for_service( + opctx, + ExternalServiceKind::Nexus, + &DataPageParams::max_page(), + ) + .await + .expect("listing Nexus's pools should succeed"); + assert!(pools.is_empty()); + assert_eq!(pool_rcgen(datastore, pool_id).await, rcgen_before); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_list_and_pagination() { + let logctx = dev::test_setup_log("test_list_and_pagination"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let pool1 = create_service_ip_pool( + opctx, + datastore, + "svc-pool-1", + IpVersion::V4, + ) + .await; + let pool2 = create_service_ip_pool( + opctx, + datastore, + "svc-pool-2", + IpVersion::V4, + ) + .await; + let pool3 = create_service_ip_pool( + opctx, + datastore, + "svc-pool-3", + IpVersion::V4, + ) + .await; + let pool4 = create_service_ip_pool( + opctx, + datastore, + "svc-pool-4", + IpVersion::V4, + ) + .await; + let id1 = pool1.authz_pool.id(); + let id2 = pool2.authz_pool.id(); + let id3 = pool3.authz_pool.id(); + let id4 = pool4.authz_pool.id(); + + // Nexus: {pool1, pool2} + // boundary NTP: {pool2, pool3} (pool2 shared) + // External DNS: {pool4} + let assignments = [ + (ExternalServiceKind::Nexus, &pool1), + (ExternalServiceKind::Nexus, &pool2), + (ExternalServiceKind::BoundaryNtp, &pool2), + (ExternalServiceKind::BoundaryNtp, &pool3), + (ExternalServiceKind::ExternalDns, &pool4), + ]; + for (service, pool) in assignments { + datastore + .external_service_ip_pool_assign( + opctx, + service, + &pool.authz_pool, + ) + .await + .expect("assigning should succeed"); + } + + // List for each service, ensure it's sorted by ID. + let mut nexus_pools = datastore + .external_service_ip_pool_pools_for_service( + opctx, + ExternalServiceKind::Nexus, + &DataPageParams::max_page(), + ) + .await + .expect("listing Nexus's pools should succeed"); + nexus_pools.sort(); + let mut expected_nexus_pools = vec![id1, id2]; + expected_nexus_pools.sort(); + assert_eq!(nexus_pools, expected_nexus_pools); + + let mut ntp_pools = datastore + .external_service_ip_pool_pools_for_service( + opctx, + ExternalServiceKind::BoundaryNtp, + &DataPageParams::max_page(), + ) + .await + .expect("listing boundary NTP's pools should succeed"); + ntp_pools.sort(); + let mut expected_ntp_pools = vec![id2, id3]; + expected_ntp_pools.sort(); + assert_eq!(ntp_pools, expected_ntp_pools); + + // DNS only got one pool, no sorting needed. + let dns_pools = datastore + .external_service_ip_pool_pools_for_service( + opctx, + ExternalServiceKind::ExternalDns, + &DataPageParams::max_page(), + ) + .await + .expect("listing external DNS's pools should succeed"); + assert_eq!(dns_pools, &[id4]); + + // Walk the full list in pages of 2 to exercise pagination. + let mut seen: Vec<(ExternalServiceKind, Uuid)> = Vec::new(); + let mut paginator = Paginator::new( + 2.try_into().unwrap(), + dropshot::PaginationOrder::Ascending, + ); + while let Some(p) = paginator.next() { + let batch = datastore + .external_service_ip_pool_list(opctx, &p.current_pagparams()) + .await + .expect("listing a page of assignments should succeed"); + paginator = + p.found_batch(&batch, &|row: &ExternalServiceIpPool| { + (row.service, row.ip_pool_id.into_untyped_uuid()) + }); + seen.extend( + batch.into_iter().map(|row| { + (row.service, row.ip_pool_id.into_untyped_uuid()) + }), + ); + } + assert_eq!(seen.len(), 5); + for expected in [ + (ExternalServiceKind::Nexus, id1), + (ExternalServiceKind::Nexus, id2), + (ExternalServiceKind::BoundaryNtp, id2), + (ExternalServiceKind::BoundaryNtp, id3), + (ExternalServiceKind::ExternalDns, id4), + ] { + assert!(seen.contains(&expected), "list is missing {expected:?}"); + } + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_cannot_delete_pool_assigned_to_service() { + let logctx = + dev::test_setup_log("test_cannot_delete_pool_assigned_to_service"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create two pools for services, so we don't hit the "at least one + // pool" check in the `ip_pool_delete()` method. + let pool1 = create_service_ip_pool( + opctx, + datastore, + "svc-pool-1", + IpVersion::V4, + ) + .await; + let _pool2 = create_service_ip_pool( + opctx, + datastore, + "svc-pool-2", + IpVersion::V4, + ) + .await; + + datastore + .external_service_ip_pool_assign( + opctx, + ExternalServiceKind::Nexus, + &pool1.authz_pool, + ) + .await + .expect("assigning should succeed"); + + // Still assigned to Nexus, can't delete the pool at all. + let err = datastore + .ip_pool_delete(opctx, &pool1.authz_pool, &pool1.db_pool) + .await + .expect_err("deleting a pool assigned to a service should fail"); + assert_matches!(err, Error::InvalidRequest { .. }); + + db.terminate().await; + logctx.cleanup_successful(); + } + + const POOL_ID: Uuid = uuid::uuid!("bce2bca9-2183-4625-9560-4dfab8378c1e"); + + #[tokio::test] + async fn expectorate_assign_pool_to_service_query() { + let query = + assign_pool_to_service_query(ExternalServiceKind::Nexus, POOL_ID); + expectorate_query_contents( + &query, + "tests/output/external_service_ip_pool_assign.sql", + ) + .await; + } + + #[tokio::test] + async fn expectorate_unassign_pool_from_service_query() { + let query = unassign_pool_from_service_query( + ExternalServiceKind::Nexus, + POOL_ID, + ); + expectorate_query_contents( + &query, + "tests/output/external_service_ip_pool_unassign.sql", + ) + .await; + } +} diff --git a/nexus/db-queries/src/db/datastore/ip_pool.rs b/nexus/db-queries/src/db/datastore/ip_pool.rs index 49678e71cf1..cf19c2dcb12 100644 --- a/nexus/db-queries/src/db/datastore/ip_pool.rs +++ b/nexus/db-queries/src/db/datastore/ip_pool.rs @@ -28,6 +28,7 @@ use crate::db::queries::ip_pool::FilterOverlappingIpRanges; use crate::db::raw_query_builder::QueryBuilder; use crate::db::raw_query_builder::SelectableSql; use crate::db::raw_query_builder::TypedSqlQuery; +use crate::db::true_or_cast_error::matches_sentinel; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::AggregateExpressionMethods; @@ -121,6 +122,13 @@ const DELETE_LAST_POOL_ERROR: &str = "Cannot delete the last IP Pool \ assigned for system services use. Create and assign at least one more \ IP Pool before deleting this one."; +// Error message emitted when a user attempts to assign a pool from system +// serivces to silos, and that pool is currently assigned to a specific external +// service, e.g., Nexus. +const ASSIGNED_TO_EXTERNAL_SERVICE_ERROR: &str = "Cannot reassign an IP \ + Pool for silos, while it is assigned to a specific external service. \ + Unassign it first, and try again."; + /// Check if pool selection has an IP version conflict. /// /// When `ip_version` is not specified and multiple pools exist with different @@ -834,6 +842,7 @@ impl DataStore { authz_pool: &authz::IpPool, db_pool: &IpPool, ) -> DeleteResult { + use nexus_db_schema::schema::external_service_ip_pool; use nexus_db_schema::schema::ip_pool::dsl; use nexus_db_schema::schema::ip_pool_range; use nexus_db_schema::schema::ip_pool_resource; @@ -857,6 +866,29 @@ impl DataStore { )); } + // Verify the pool is not assigned to any external service. A concurrent + // assignment is caught separately by the `rcgen` check on the delete + // below, since assigning a pool to a service bumps its `rcgen`. + let assignment = + external_service_ip_pool::dsl::external_service_ip_pool + .filter( + external_service_ip_pool::dsl::ip_pool_id + .eq(authz_pool.id()), + ) + .select(external_service_ip_pool::dsl::ip_pool_id) + .limit(1) + .first_async::(&*conn) + .await + .optional() + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + if assignment.is_some() { + return Err(Error::invalid_request( + "IP Pool cannot be deleted while it is assigned to a service", + )); + } + // Add a small subquery, if needed, to ensure that we don't delete this // IP Pool if it's the last system-services pool. There has to always be at // least one of these. @@ -964,8 +996,18 @@ impl DataStore { ref info, ) => { let message = info.message(); - if message.ends_with("invalid bool value") { + if matches_sentinel(&e, &[BAD_SILO_LINK_SENTINEL]).is_some() + { Error::invalid_request(BAD_SILO_LINK_ERROR) + } else if matches_sentinel( + &e, + &[ASSIGNED_TO_EXTERNAL_SERVICE_SENTINEL], + ) + .is_some() + { + Error::invalid_request( + ASSIGNED_TO_EXTERNAL_SERVICE_ERROR, + ) } else if message.contains("division by zero") { Error::invalid_request(POOL_HAS_IPS_ERROR) } else if message.starts_with("could not parse") @@ -2198,6 +2240,11 @@ const SILO_DELETED_SENTINEL: &str = "silo-deleted"; // system-services pool of a given type. const LAST_POOL_SENTINEL: &str = "last-pool"; +// Sentinel we try to cast as a bool when a user attempts to assign a pool from +// system serivces to silos, and that pool is currently assigned to a specific +// external service, e.g., Nexus. +const ASSIGNED_TO_EXTERNAL_SERVICE_SENTINEL: &str = "assigned-to-services"; + /// Extract the sentinel string from a UUID cast error message. /// /// When our SQL queries intentionally fail a UUID cast to signal an error @@ -2574,6 +2621,19 @@ fn reassign_system_services_ip_pool_query( LIMIT 1\ ), 1/0, 1) AS BOOL))", ) + // Generate bool-cast error if the pool is currently assigned to a + // specific external service like Nexus. + .sql( + " AND CAST(IF(EXISTS(\ + SELECT 1 \ + FROM external_service_ip_pool \ + WHERE ip_pool_id = ", + ) + .param() + .bind::(ip_pool.id()) + .sql("), '") + .sql(ASSIGNED_TO_EXTERNAL_SERVICE_SENTINEL) + .sql("', 'TRUE') AS BOOL)") // Generate int-cast error if this is the last pool of this assignment. .sql( " AND CAST(IF(\ @@ -2609,9 +2669,9 @@ mod test { use crate::authz; use crate::db::datastore::external_ip::FloatingIpAllocation; use crate::db::datastore::ip_pool::{ - BAD_SILO_LINK_ERROR, DELETE_LAST_POOL_ERROR, POOL_HAS_IPS_ERROR, - REASSIGN_LAST_POOL_ERROR, assign_ip_pool_query, - link_ip_pool_to_external_silo_query, + ASSIGNED_TO_EXTERNAL_SERVICE_ERROR, BAD_SILO_LINK_ERROR, + DELETE_LAST_POOL_ERROR, POOL_HAS_IPS_ERROR, REASSIGN_LAST_POOL_ERROR, + assign_ip_pool_query, link_ip_pool_to_external_silo_query, unlink_ip_pool_from_external_silo_query, }; use crate::db::explain::ExplainableAsync as _; @@ -2630,8 +2690,8 @@ mod test { }; use nexus_db_lookup::LookupPath; use nexus_db_model::{ - InternetGatewayIpPool, IpPoolAssignment, IpPoolIdentity, IpPoolType, - IpVersion, + ExternalServiceKind, InternetGatewayIpPool, IpPoolAssignment, + IpPoolIdentity, IpPoolType, IpVersion, }; use nexus_types::deployment::{ OmicronZoneExternalFloatingIp, OmicronZoneExternalIp, @@ -6303,4 +6363,107 @@ mod test { db.terminate().await; logctx.cleanup_successful(); } + + #[tokio::test] + async fn cannot_assign_ip_pool_to_silos_while_assigned_to_services() { + let logctx = dev::test_setup_log( + "cannot_assign_ip_pool_to_silos_while_assigned_to_services", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create the v4 system-service pool, add a range, and assign it to an + // external service. Also create a v6 pool so that reassigning the v4 + // pool for silo use is allowed at the end (more than one + // system-service pool remains). + let ipv4 = create_service_ip_pool( + opctx, + datastore, + "oxide-service-pool-v4", + IpVersion::V4.into(), + ) + .await; + let ipv6 = create_service_ip_pool( + opctx, + datastore, + "oxide-service-pool-v6", + IpVersion::V6.into(), + ) + .await; + let ip_range = IpRange::V4(Ipv4Range { + first: Ipv4Addr::new(1, 1, 1, 1), + last: Ipv4Addr::new(1, 1, 1, 10), + }); + datastore + .ip_pool_add_range( + opctx, + &ipv4.authz_pool, + &ipv4.db_pool, + &ip_range, + ) + .await + .unwrap(); + + // Assign both to Nexus. + datastore + .external_service_ip_pool_assign( + opctx, + ExternalServiceKind::Nexus, + &ipv4.authz_pool, + ) + .await + .expect("Should be able to assign pool to Nexus"); + datastore + .external_service_ip_pool_assign( + opctx, + ExternalServiceKind::Nexus, + &ipv6.authz_pool, + ) + .await + .expect("Should be able to assign pool to Nexus"); + + // Should not be able to assign the IPv4 pool to silos now, since + // it's assigned for use by Nexus. + let res = datastore + .ip_pool_assign( + opctx, + &ipv4.authz_pool, + &ipv4.db_pool, + IpPoolAssignment::Silos, + ) + .await; + let Err(Error::InvalidRequest { message }) = &res else { + panic!( + "Should not be able to assign to silos internal \ + IP Pool when assigned to Nexus, found {res:#?}" + ); + }; + assert_eq!( + message.external_message(), + ASSIGNED_TO_EXTERNAL_SERVICE_ERROR + ); + + // Unassign it, and now we can assign the pool for silo use. + let _ = datastore + .external_service_ip_pool_unassign( + opctx, + ExternalServiceKind::Nexus, + &ipv4.authz_pool, + ) + .await + .expect("Should be able to unassign IP Pool"); + let _ = datastore.ip_pool_assign( + opctx, + &ipv4.authz_pool, + &ipv4.db_pool, + IpPoolAssignment::Silos, + ).await + .expect( + "Should be able to delete internal IP Pool when more than one remains, \ + after unassigning it from Nexus" + ); + + db.terminate().await; + logctx.cleanup_successful(); + } } diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index 69f480e2938..3443ac5c2e8 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -94,6 +94,7 @@ mod disk; mod dns; mod ereport; mod external_ip; +mod external_service_ip_pool; mod external_subnet; pub mod fm; mod fm_config; diff --git a/nexus/db-queries/tests/output/assign_ip_pool_to_system_services.sql b/nexus/db-queries/tests/output/assign_ip_pool_to_system_services.sql index 94abe96cd26..1ae63ce6b13 100644 --- a/nexus/db-queries/tests/output/assign_ip_pool_to_system_services.sql +++ b/nexus/db-queries/tests/output/assign_ip_pool_to_system_services.sql @@ -28,9 +28,17 @@ WHERE ) AND CAST( IF( - (SELECT count(1) FROM ip_pool WHERE time_deleted IS NULL AND assignment = $5 LIMIT 2) >= 2, + EXISTS(SELECT 1 FROM external_service_ip_pool WHERE ip_pool_id = $5), + 'assigned-to-services', + 'TRUE' + ) + AS BOOL + ) + AND CAST( + IF( + (SELECT count(1) FROM ip_pool WHERE time_deleted IS NULL AND assignment = $6 LIMIT 2) >= 2, '1', - $6 + $7 ) AS INT8 ) diff --git a/nexus/db-queries/tests/output/external_service_ip_pool_assign.sql b/nexus/db-queries/tests/output/external_service_ip_pool_assign.sql new file mode 100644 index 00000000000..1606a890b02 --- /dev/null +++ b/nexus/db-queries/tests/output/external_service_ip_pool_assign.sql @@ -0,0 +1,21 @@ +WITH + bumped + AS ( + UPDATE + ip_pool + SET + rcgen = rcgen + 1 + WHERE + id = $1 + AND time_deleted IS NULL + AND CAST(IF(assignment = $2, 'true', 'not-a-system-services-pool') AS BOOL) + RETURNING + id + ) +INSERT +INTO + external_service_ip_pool (service, ip_pool_id) +SELECT + $3, id +FROM + bumped diff --git a/nexus/db-queries/tests/output/external_service_ip_pool_unassign.sql b/nexus/db-queries/tests/output/external_service_ip_pool_unassign.sql new file mode 100644 index 00000000000..59c963a76b3 --- /dev/null +++ b/nexus/db-queries/tests/output/external_service_ip_pool_unassign.sql @@ -0,0 +1,25 @@ +WITH + deleted + AS ( + DELETE FROM + external_service_ip_pool + WHERE + service = $1 + AND ip_pool_id = $2 + AND CAST( + IF( + (SELECT count(1) FROM external_service_ip_pool WHERE service = $3) >= 2, + 'true', + 'unassign-last-pool' + ) + AS BOOL + ) + RETURNING + ip_pool_id + ) +UPDATE + ip_pool +SET + rcgen = rcgen + 1 +WHERE + id = $4 AND EXISTS(SELECT 1 FROM deleted) diff --git a/nexus/db-schema/src/enums.rs b/nexus/db-schema/src/enums.rs index 750297a269d..97ab39df181 100644 --- a/nexus/db-schema/src/enums.rs +++ b/nexus/db-schema/src/enums.rs @@ -57,6 +57,7 @@ define_enums! { DownstairsClientStopRequestReasonEnum => "downstairs_client_stop_request_reason_type", DownstairsClientStoppedReasonEnum => "downstairs_client_stopped_reason_type", EreporterTypeEnum => "ereporter_type", + ExternalServiceKindEnum => "external_service_kind", FailureDomainEnum => "failure_domain", FmFactPhysicalDiskKindEnum => "fm_fact_physical_disk_kind", FmFactSagaKindEnum => "fm_fact_saga_kind", diff --git a/nexus/db-schema/src/schema.rs b/nexus/db-schema/src/schema.rs index 25ab2c72802..f3c79575e44 100644 --- a/nexus/db-schema/src/schema.rs +++ b/nexus/db-schema/src/schema.rs @@ -670,6 +670,13 @@ table! { } } +table! { + external_service_ip_pool (service, ip_pool_id) { + service -> crate::enums::ExternalServiceKindEnum, + ip_pool_id -> Uuid, + } +} + table! { ip_pool_resource (ip_pool_id, resource_type, resource_id) { ip_pool_id -> Uuid, diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 0c045c570de..a1d8738056b 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -2378,6 +2378,40 @@ CREATE INDEX IF NOT EXISTS lookup_ip_pool_by_type ON omicron.public.ip_pool ( ) WHERE time_deleted IS NULL; +/* + * The external services whose external addresses are drawn from IP pools. + */ +CREATE TYPE IF NOT EXISTS omicron.public.external_service_kind AS ENUM ( + 'nexus', + 'boundary_ntp', + 'external_dns' +); + +/* + * Join table assigning IP pools to external services. + * + * This represents the operator's intent about which pools should be used for + * those services, but we're intentionally not specifying the semantics yet + * (e.g., an assignment means we MUST use an IP from the pool, vs MAY do so). + * We'll flesh that out per-service as needed during implementation. + */ +CREATE TABLE IF NOT EXISTS omicron.public.external_service_ip_pool ( + service omicron.public.external_service_kind NOT NULL, + ip_pool_id UUID NOT NULL, + -- Most commonly want to look up per-service first, e.g., during blueprint + -- planning + PRIMARY KEY (service, ip_pool_id) +); + +/* + * Index supporting fast lookup of a pool, e.g. to list services assigned to it. + * Also used when deleting the actual `ip_pool` row, failing the query if it's + * still assigned to anything. + */ +CREATE INDEX IF NOT EXISTS external_service_ip_pool_by_ip_pool_id ON omicron.public.external_service_ip_pool ( + ip_pool_id +); + -- The order here is most-specific first, and it matters because we use this -- fact to select the most specific default in the case where there is both a -- silo default and a fleet default. If we were to add a project type, it should @@ -9353,7 +9387,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '293.0.0', NULL) + (TRUE, NOW(), NOW(), '294.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/crdb/external-service-ip-pool/up01.sql b/schema/crdb/external-service-ip-pool/up01.sql new file mode 100644 index 00000000000..65409fcdfed --- /dev/null +++ b/schema/crdb/external-service-ip-pool/up01.sql @@ -0,0 +1,5 @@ +CREATE TYPE IF NOT EXISTS omicron.public.external_service_kind AS ENUM ( + 'nexus', + 'boundary_ntp', + 'external_dns' +); diff --git a/schema/crdb/external-service-ip-pool/up02.sql b/schema/crdb/external-service-ip-pool/up02.sql new file mode 100644 index 00000000000..ef46f1a6a3c --- /dev/null +++ b/schema/crdb/external-service-ip-pool/up02.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS omicron.public.external_service_ip_pool ( + service omicron.public.external_service_kind NOT NULL, + ip_pool_id UUID NOT NULL, + + PRIMARY KEY (service, ip_pool_id) +); diff --git a/schema/crdb/external-service-ip-pool/up03.sql b/schema/crdb/external-service-ip-pool/up03.sql new file mode 100644 index 00000000000..b518598f4be --- /dev/null +++ b/schema/crdb/external-service-ip-pool/up03.sql @@ -0,0 +1,3 @@ +CREATE INDEX IF NOT EXISTS external_service_ip_pool_by_ip_pool_id ON omicron.public.external_service_ip_pool ( + ip_pool_id +); diff --git a/schema/crdb/external-service-ip-pool/up03.verify.sql b/schema/crdb/external-service-ip-pool/up03.verify.sql new file mode 100644 index 00000000000..0df7a29413d --- /dev/null +++ b/schema/crdb/external-service-ip-pool/up03.verify.sql @@ -0,0 +1,2 @@ +-- DO NOT EDIT. Generated by test_migration_verification_files. +SELECT CAST(IF((SELECT true WHERE EXISTS (SELECT index_name FROM omicron.crdb_internal.table_indexes WHERE descriptor_name = 'external_service_ip_pool' AND index_name = 'external_service_ip_pool_by_ip_pool_id')),'true','Schema change verification failed: index external_service_ip_pool_by_ip_pool_id on table external_service_ip_pool does not exist') AS BOOL); diff --git a/uuid-kinds/src/lib.rs b/uuid-kinds/src/lib.rs index df2e379bed6..9d2bd300fb8 100644 --- a/uuid-kinds/src/lib.rs +++ b/uuid-kinds/src/lib.rs @@ -67,6 +67,7 @@ impl_typed_uuid_kinds! { FmdResource = {}, Instance = {}, InternalZpool = {}, + IpPool = {}, LoopbackAddress = {}, MulticastGroup = {}, Measurement = {},