feat: add logic to handle updates to operator config (#1186)

## Description

This PR adds logic to handle updates to the operator config,
specifically managed via the operator secret. In particular this adds:
- A watch on the operator config secret
- A function to manage changes to that secret
- Logic to specifically handle additions/changes/deletions of the
authservice config (redis and ca cert)
- Logic to handle updates to the CIDR ranges for nodes/kubeapi
- Stubbed out logic to handle updates to domain and admin domain (this
would be a larger effort to fully implement so leaving a TODO on this)

## Related Issue

Fixes #1130

## Type of change

- [x] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Other (security config, docs update, etc)

## Steps to Validate

<details>

```console
# Deploy slim-dev with unicorn flavor (or other flavor)
uds run slim-dev --set flavor=unicorn

# Validate an update to redis URI takes effect
## Edit the operator secret
kubectl edit secret -n pepr-system uds-operator-config
## Add `cmVkaXM6Ly90ZXN0` as the value for `AUTHSERVICE_REDIS_URI` (this is base64 encoded `redis://test`)
## Validate that Pepr watcher logs indicate it cycled authservice
kubectl logs -n pepr-system deploy/pepr-uds-core-watcher | grep "Updating Authservice secret"
## Validate that Authservice cycled and is trying to use our fake redis url
kubectl logs -n authservice deploy/authservice --all-pods # You should see `redis-url="redis://test"`
## Edit the operator secret
kubectl edit secret -n pepr-system uds-operator-config
## Remove the `AUTHSERVICE_REDIS_URI` value (change to empty string)
## Validate that Pepr watcher logs indicate it cycled authservice
kubectl logs -n pepr-system deploy/pepr-uds-core-watcher | grep "Updating Authservice secret"
## Validate that Authservice cycled and is no longer using redis
kubectl logs -n authservice deploy/authservice --all-pods # It shouldn't have redis listed

# You could redo these same steps with a CA Cert value, just make sure that the value you use is "double" base64 encoded since we expect it to be base64 encoded, and then the secret requires it to be base64 encoded as well

# Validate updates to the CIDR values
## Deploy monitoring layer to add some test resources
uds run test:single-layer --set LAYER=monitoring --set FLAVOR=unicorn
## Edit the operator secret
kubectl edit secret -n pepr-system uds-operator-config
## Set both `KUBEAPI_CIDR` and `KUBENODE_CIDRS` to `MTkyLjE2OC4wLjEvMzI=` (this is `192.168.0.1/32` base64 encoded)
## Validate that Pepr watcher logs indicate it updated network policies
kubectl logs -n pepr-system deploy/pepr-uds-core-watcher | grep "Updating KubeNodes"
kubectl logs -n pepr-system deploy/pepr-uds-core-watcher | grep "Updating KubeAPI"
## Validate the network policies actually have our new IP (`192.168.0.1/32`)
kubectl get networkpolicy -n monitoring allow-prometheus-stack-egress-kube-prometheus-stack-admission-create-kubeapi -o yaml
kubectl get networkpolicy -n monitoring allow-prometheus-stack-egress-metrics-scraping-of-kube-nodes -o yaml
```

</details>

## Checklist before merging

- [x] Test, docs, adr added or updated as needed
- [x] [Contributor
Guide](https://github.com/defenseunicorns/uds-template-capability/blob/main/CONTRIBUTING.md)
followed

---------

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: Noah <[email protected]>
Co-authored-by: Chance <[email protected]>

src/pepr/logger.ts

@@ -10,6 +10,7 @@ export enum Component {
   CONFIG = "config",
   ISTIO = "istio",
   OPERATOR = "operator",
+  OPERATOR_CONFIG = "operator.config",
   OPERATOR_EXEMPTIONS = "operator.exemptions",
   OPERATOR_ISTIO = "operator.istio",
   OPERATOR_KEYCLOAK = "operator.keycloak",

src/pepr/operator/controllers/config/config.spec.ts

@@ -0,0 +1,197 @@
+/**
+ * Copyright 2025 Defense Unicorns
+ * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+ */
+
+import { beforeEach, describe, expect, it, jest } from "@jest/globals";
+import { kind } from "pepr";
+import { UDSConfig } from "../../../config";
+import { Component, setupLogger } from "../../../logger";
+import { reconcileAuthservice } from "../keycloak/authservice/authservice";
+import { initAPIServerCIDR } from "../network/generators/kubeAPI";
+import { initAllNodesTarget } from "../network/generators/kubeNodes";
+import { updateUDSConfig } from "./config";
+
+// Mock dependencies
+jest.mock("../../../config", () => ({
+  UDSConfig: {
+    caCert: "",
+    authserviceRedisUri: "",
+    kubeApiCidr: "",
+    kubeNodeCidrs: "",
+    domain: "",
+    adminDomain: "",
+    allowAllNSExemptions: false,
+  },
+}));
+
+jest.mock("../keycloak/authservice/authservice", () => ({
+  reconcileAuthservice: jest.fn(),
+}));
+
+jest.mock("../network/generators/kubeAPI", () => ({
+  initAPIServerCIDR: jest.fn(),
+}));
+
+jest.mock("../network/generators/kubeNodes", () => ({
+  initAllNodesTarget: jest.fn(),
+}));
+
+jest.mock("../../../logger", () => {
+  const mockLogger = {
+    warn: jest.fn(),
+    level: jest.fn(),
+    fatal: jest.fn(),
+    error: jest.fn(),
+    info: jest.fn(),
+    debug: jest.fn(),
+    trace: jest.fn(),
+  };
+  return {
+    Component: {
+      OPERATOR_CONFIG: "operator-config",
+    },
+    setupLogger: jest.fn(() => mockLogger),
+  };
+});
+
+describe("updateUDSConfig", () => {
+  let mockSecret: kind.Secret;
+
+  beforeEach(() => {
+    mockSecret = {
+      data: {
+        // This is "double base64 encoded" because the user will provide
+        // a base64 encoded CA cert, which is then base64 encoded again for the k8s secret
+        UDS_CA_CERT: btoa(btoa("mock-ca-cert")),
+        AUTHSERVICE_REDIS_URI: btoa("mock-redis-uri"),
+        KUBEAPI_CIDR: btoa("mock-cidr"),
+        KUBENODE_CIDRS: btoa("mock-node-cidrs"),
+        UDS_DOMAIN: btoa("mock-domain"),
+        UDS_ADMIN_DOMAIN: btoa("mock-admin-domain"),
+        UDS_ALLOW_ALL_NS_EXEMPTIONS: btoa("true"),
+      },
+    };
+
+    // Reset mocks
+    jest.clearAllMocks();
+    UDSConfig.caCert = "";
+    UDSConfig.authserviceRedisUri = "";
+    UDSConfig.kubeApiCidr = "";
+    UDSConfig.kubeNodeCidrs = "";
+    UDSConfig.domain = "";
+    UDSConfig.adminDomain = "";
+    UDSConfig.allowAllNSExemptions = false;
+  });
+
+  it("should decode the secret data and update UDSConfig", async () => {
+    await updateUDSConfig(mockSecret);
+
+    expect(UDSConfig.caCert).toBe(btoa("mock-ca-cert"));
+    expect(UDSConfig.authserviceRedisUri).toBe("mock-redis-uri");
+    expect(UDSConfig.kubeApiCidr).toBe("mock-cidr");
+    expect(UDSConfig.kubeNodeCidrs).toBe("mock-node-cidrs");
+    expect(UDSConfig.domain).toBe("mock-domain");
+    expect(UDSConfig.adminDomain).toBe("mock-admin-domain");
+    expect(UDSConfig.allowAllNSExemptions).toBe(true);
+  });
+
+  it("should call reconcileAuthservice if CA Cert or Redis URI changes", async () => {
+    UDSConfig.caCert = "old-ca-cert";
+    UDSConfig.authserviceRedisUri = "old-redis-uri";
+
+    await updateUDSConfig(mockSecret);
+
+    expect(reconcileAuthservice).toHaveBeenCalledWith({
+      name: "global-config-update",
+      action: expect.any(String),
+      trustedCA: "mock-ca-cert",
+      redisUri: "mock-redis-uri",
+    });
+  });
+
+  it("should call initAPIServerCIDR if KUBEAPI_CIDR changes", async () => {
+    UDSConfig.kubeApiCidr = "old-cidr";
+
+    await updateUDSConfig(mockSecret);
+
+    expect(initAPIServerCIDR).toHaveBeenCalled();
+  });
+
+  it("should call initAllNodesTarget if KUBENODE_CIDRS changes", async () => {
+    UDSConfig.kubeNodeCidrs = "old-node-cidrs";
+
+    await updateUDSConfig(mockSecret);
+
+    expect(initAllNodesTarget).toHaveBeenCalled();
+  });
+
+  it("should update domain and adminDomain with fallback values if unset", async () => {
+    if (mockSecret.data) {
+      mockSecret.data.UDS_DOMAIN = btoa("###ZARF_VAR_DOMAIN###");
+      mockSecret.data.UDS_ADMIN_DOMAIN = btoa("###ZARF_VAR_ADMIN_DOMAIN###");
+    }
+
+    await updateUDSConfig(mockSecret);
+
+    expect(UDSConfig.domain).toBe("uds.dev");
+    expect(UDSConfig.adminDomain).toBe("admin.uds.dev");
+  });
+
+  it("should not call unnecessary updates if no values change", async () => {
+    // Set UDSConfig to match mockSecret data
+    UDSConfig.caCert = btoa("mock-ca-cert");
+    UDSConfig.authserviceRedisUri = "mock-redis-uri";
+    UDSConfig.kubeApiCidr = "mock-cidr";
+    UDSConfig.kubeNodeCidrs = "mock-node-cidrs";
+    UDSConfig.domain = "mock-domain";
+    UDSConfig.adminDomain = "mock-admin-domain";
+    UDSConfig.allowAllNSExemptions = true;
+
+    await updateUDSConfig(mockSecret);
+
+    expect(reconcileAuthservice).not.toHaveBeenCalled();
+    expect(initAPIServerCIDR).not.toHaveBeenCalled();
+    expect(initAllNodesTarget).not.toHaveBeenCalled();
+  });
+
+  it("should not call netpol updates if no values change", async () => {
+    // Set mockSecret to match UDSConfig data
+    mockSecret = {
+      data: {
+        UDS_CA_CERT: btoa(btoa("mock-ca-cert")),
+        AUTHSERVICE_REDIS_URI: btoa("mock-redis-uri"),
+        KUBEAPI_CIDR: "",
+        KUBENODE_CIDRS: "",
+        UDS_DOMAIN: btoa("mock-domain"),
+        UDS_ADMIN_DOMAIN: btoa("mock-admin-domain"),
+        UDS_ALLOW_ALL_NS_EXEMPTIONS: btoa("true"),
+      },
+    };
+    await updateUDSConfig(mockSecret);
+
+    expect(initAPIServerCIDR).not.toHaveBeenCalled();
+    expect(initAllNodesTarget).not.toHaveBeenCalled();
+  });
+
+  it("should set caCert to an empty string if the value is a placeholder", async () => {
+    if (mockSecret.data) {
+      mockSecret.data.UDS_CA_CERT = btoa("###ZARF_VAR_CA_CERT###");
+    }
+    await updateUDSConfig(mockSecret);
+    expect(UDSConfig.caCert).toBe("");
+  });
+
+  it("should log an error and set caCert to an empty string if the value is not valid base64", async () => {
+    if (mockSecret.data) {
+      mockSecret.data.UDS_CA_CERT = btoa("invalid-base64");
+    }
+    const mockLogger = setupLogger(Component.OPERATOR_CONFIG);
+
+    await updateUDSConfig(mockSecret);
+    expect(mockLogger.error).toHaveBeenCalledWith(
+      expect.stringContaining("Invalid CA Cert provided in uds-operator-config secret"),
+    );
+    expect(UDSConfig.caCert).toBe("");
+  });
+});

src/pepr/operator/controllers/config/config.ts

@@ -0,0 +1,111 @@
+/**
+ * Copyright 2025 Defense Unicorns
+ * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+ */
+
+import { kind } from "pepr";
+import { UDSConfig } from "../../../config";
+import { Component, setupLogger } from "../../../logger";
+import { reconcileAuthservice } from "../keycloak/authservice/authservice";
+import { Action, AuthServiceEvent } from "../keycloak/authservice/types";
+import { initAPIServerCIDR } from "../network/generators/kubeAPI";
+import { initAllNodesTarget } from "../network/generators/kubeNodes";
+
+// configure subproject logger
+const log = setupLogger(Component.OPERATOR_CONFIG);
+
+export async function updateUDSConfig(config: kind.Secret) {
+  log.info("Updating UDS Config from uds-operator-config secret change");
+
+  // Base64 decode the secret data
+  const decodedConfigData: { [key: string]: string } = {};
+  for (const key in config.data) {
+    try {
+      const decodedValue = atob(config.data[key]);
+      if (decodedValue) {
+        decodedConfigData[key] = decodedValue;
+      } else {
+        decodedConfigData[key] = "";
+      }
+    } catch (e) {
+      log.error(`Failed to decode secret key: ${key}, error: ${e.message}`);
+    }
+  }
+
+  // Handle changes to the Authservice configuration
+  if (
+    decodedConfigData.UDS_CA_CERT !== UDSConfig.caCert ||
+    decodedConfigData.AUTHSERVICE_REDIS_URI !== UDSConfig.authserviceRedisUri
+  ) {
+    UDSConfig.caCert = decodedConfigData.UDS_CA_CERT;
+    UDSConfig.authserviceRedisUri = decodedConfigData.AUTHSERVICE_REDIS_URI;
+
+    // Account for undefined or placeholder values (dev mode)
+    if (
+      !UDSConfig.authserviceRedisUri ||
+      UDSConfig.authserviceRedisUri === "###ZARF_VAR_AUTHSERVICE_REDIS_URI###"
+    ) {
+      UDSConfig.authserviceRedisUri = "";
+    }
+    if (!UDSConfig.caCert || UDSConfig.caCert === "###ZARF_VAR_CA_CERT###") {
+      UDSConfig.caCert = "";
+    }
+    // Validate that the cacert is base64 encoded (it should be)
+    if (UDSConfig.caCert) {
+      try {
+        atob(UDSConfig.caCert);
+      } catch (e) {
+        log.error(
+          "Invalid CA Cert provided in uds-operator-config secret, falling back to no CA Cert",
+        );
+        UDSConfig.caCert = "";
+      }
+    }
+
+    const authserviceUpdate: AuthServiceEvent = {
+      name: "global-config-update",
+      action: Action.UpdateGlobalConfig,
+      // Base64 decode the CA cert before passing to the update function
+      trustedCA: atob(UDSConfig.caCert),
+      redisUri: UDSConfig.authserviceRedisUri,
+    };
+    log.debug("Updating Authservice secret based on change to CA Cert or Redis URI");
+    await reconcileAuthservice(authserviceUpdate);
+  }
+
+  // Handle changes to the kubeApiCidr
+  if (decodedConfigData.KUBEAPI_CIDR !== UDSConfig.kubeApiCidr) {
+    UDSConfig.kubeApiCidr = decodedConfigData.KUBEAPI_CIDR;
+    // This re-runs the "init" function to update netpols if necessary
+    log.debug("Updating KubeAPI network policies based on change to kubeApiCidr");
+    await initAPIServerCIDR();
+  }
+
+  // Handle changes to the kubeNodeCidrs
+  if (decodedConfigData.KUBENODE_CIDRS !== UDSConfig.kubeNodeCidrs) {
+    UDSConfig.kubeNodeCidrs = decodedConfigData.KUBENODE_CIDRS;
+    // This re-runs the "init" function to update netpols if necessary
+    log.debug("Updating KubeNodes network policies based on change to kubeNodeCidrs");
+    await initAllNodesTarget();
+  }
+
+  if (
+    decodedConfigData.UDS_DOMAIN !== UDSConfig.domain ||
+    decodedConfigData.UDS_ADMIN_DOMAIN !== UDSConfig.adminDomain
+  ) {
+    UDSConfig.domain = decodedConfigData.UDS_DOMAIN;
+    UDSConfig.adminDomain = decodedConfigData.UDS_ADMIN_DOMAIN;
+    if (!UDSConfig.domain || UDSConfig.domain === "###ZARF_VAR_DOMAIN###") {
+      UDSConfig.domain = "uds.dev";
+    }
+    if (!UDSConfig.adminDomain || UDSConfig.adminDomain === "###ZARF_VAR_ADMIN_DOMAIN###") {
+      UDSConfig.adminDomain = `admin.${UDSConfig.domain}`;
+    }
+    // todo: Add logic to handle domain changes and update across virtualservices, authservice config, etc
+  }
+
+  // Update other config values (no need for special handling)
+  UDSConfig.allowAllNSExemptions = decodedConfigData.UDS_ALLOW_ALL_NS_EXEMPTIONS === "true";
+
+  log.info("Updated UDS Config based on uds-operator-config secret changes");
+}

src/pepr/operator/controllers/keycloak/authservice/authorization-policy.ts

@@ -13,14 +13,14 @@ import {
 } from "../../../crd";
 import { getOwnerRef, purgeOrphans, sanitizeResourceName } from "../../utils";
 import { log } from "./authservice";
-import { Action as AuthServiceAction, AuthServiceEvent } from "./types";
+import { AddOrRemoveClientEvent, Action as AuthServiceAction } from "./types";
 
 const operationMap: {
-  [AuthServiceAction.Add]: "Apply";
-  [AuthServiceAction.Remove]: "Delete";
+  [AuthServiceAction.AddClient]: "Apply";
+  [AuthServiceAction.RemoveClient]: "Delete";
 } = {
-  [AuthServiceAction.Add]: "Apply",
-  [AuthServiceAction.Remove]: "Delete",
+  [AuthServiceAction.AddClient]: "Apply",
+  [AuthServiceAction.RemoveClient]: "Delete",
 };
 
 function authserviceAuthorizationPolicy(
@@ -114,7 +114,7 @@ function authNRequestAuthentication(
 }
 
 async function updatePolicy(
-  event: AuthServiceEvent,
+  event: AddOrRemoveClientEvent,
   labelSelector: { [key: string]: string },
   pkg: UDSPackage,
 ) {