From 5637f6acf0dd375bec4475bf7f9798e6f5f5e359 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 28 Apr 2026 12:01:15 +0200 Subject: [PATCH 1/3] Enhance vault sync with skip filter and destructive sync - Skip vaults containing 'fournos_skip_sync' key - Add destructive sync to delete orphaned managed secrets - Improve logging for skipped entries and deletions Co-Authored-By: Claude Sonnet 4 --- hacks/sync_vault_secrets.py | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/hacks/sync_vault_secrets.py b/hacks/sync_vault_secrets.py index b048d6c..db7e4b9 100755 --- a/hacks/sync_vault_secrets.py +++ b/hacks/sync_vault_secrets.py @@ -57,6 +57,9 @@ ANNOTATION_VAULT_ADDR = "fournos.dev/vault-addr" ANNOTATION_VAULT_PATH = "fournos.dev/vault-path" MANAGER_VALUE = "fournos-vault-sync" +SKIP_SYNC_KEY = ( + "fournos_skip_sync" # vaults with this key won't be synchronized to the cluster +) # --------------------------------------------------------------------------- @@ -165,6 +168,31 @@ def _apply_secret( logger.info("Updated Secret %s/%s", namespace, name) +def _list_managed_secrets(v1, namespace: str) -> list[str]: + """List all secrets managed by fournos-vault-sync.""" + from kubernetes.client.exceptions import ApiException + + try: + label_selector = f"{LABEL_MANAGED_BY}={MANAGER_VALUE},{LABEL_VAULT_ENTRY}=true" + secrets = v1.list_namespaced_secret(namespace, label_selector=label_selector) + return [secret.metadata.name for secret in secrets.items] + except ApiException as exc: + logger.error("Failed to list managed secrets: %s", exc) + return [] + + +def _delete_secret(v1, name: str, namespace: str): + """Delete a Kubernetes Secret.""" + from kubernetes.client.exceptions import ApiException + + try: + v1.delete_namespaced_secret(name, namespace) + logger.info("Deleted Secret %s/%s", namespace, name) + except ApiException as exc: + if exc.status != 404: + logger.error("Failed to delete Secret %s/%s: %s", namespace, name, exc) + + # --------------------------------------------------------------------------- # Core logic # --------------------------------------------------------------------------- @@ -206,11 +234,22 @@ def sync( if not names: logger.warning("No vault entries found under %s/%s", kv_mount, secret_path) return 0 + logger.info("Found %d vault entries: %s", len(names), ", ".join(names)) v1 = None if dry_run else _k8s_core_api() + # Track which secrets we process for destructive sync + processed_secrets = set() + + # Get list of existing managed secrets for destructive sync + existing_managed_secrets = set() + if not dry_run: + existing_managed_secrets = set(_list_managed_secrets(v1, namespace)) + logger.info("Found %d existing managed secrets", len(existing_managed_secrets)) + errors = 0 + skipped_count = 0 for vault_name in names: full_path = f"{secret_path}/{vault_name}" @@ -236,6 +275,14 @@ def sync( logger.warning("Vault entry %s is empty, skipping", full_path) continue + # Check if vault contains fournos_skip_sync key - if so, skip this vault + if SKIP_SYNC_KEY in kv_data: + logger.info( + "Skipping vault %s (contains %s key)", vault_name, SKIP_SYNC_KEY + ) + skipped_count += 1 + continue + safe_data: dict[str, str] = {} for k, v in kv_data.items(): if _is_vault_metadata(k): @@ -260,6 +307,7 @@ def sync( print(f"[dry-run] Would create/update Secret {namespace}/{secret_name}") for key in safe_data: print(f" {key}: <{len(str(safe_data[key]))} chars>") + processed_secrets.add(secret_name) continue try: @@ -271,6 +319,7 @@ def sync( vault_addr=vault_addr, vault_full_path=f"{kv_mount}/{full_path}", ) + processed_secrets.add(secret_name) except Exception: logger.exception( "Failed to apply Secret %s/%s", @@ -279,6 +328,36 @@ def sync( ) errors += 1 + # Destructive sync: delete managed secrets that are no longer in vault + secrets_to_delete = existing_managed_secrets - processed_secrets + if secrets_to_delete: + logger.info( + "Found %d managed secrets not in vault, deleting: %s", + len(secrets_to_delete), + ", ".join(secrets_to_delete), + ) + + for secret_name in secrets_to_delete: + if dry_run: + print(f"[dry-run] Would delete Secret {namespace}/{secret_name}") + else: + try: + _delete_secret(v1, secret_name, namespace) + except Exception: + logger.exception( + "Failed to delete Secret %s/%s", namespace, secret_name + ) + errors += 1 + else: + logger.info("No managed secrets need to be deleted") + + if skipped_count > 0: + logger.info( + "Total vault entries skipped (contained %s): %d", + SKIP_SYNC_KEY, + skipped_count, + ) + return 1 if errors else 0 From 683bf8afc015091a99a667988cca2fe1ac471317 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 28 Apr 2026 12:09:20 +0200 Subject: [PATCH 2/3] hacks: sync_vault_secrets: set PSAP sync default values --- hacks/sync_vault_secrets.py | 54 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/hacks/sync_vault_secrets.py b/hacks/sync_vault_secrets.py index db7e4b9..a8094c2 100755 --- a/hacks/sync_vault_secrets.py +++ b/hacks/sync_vault_secrets.py @@ -7,15 +7,15 @@ Required environment variables ------------------------------ -* ``VAULT_ADDR`` — Vault server URL (e.g. ``https://vault.example.com``) * ``VAULT_TOKEN`` — Vault authentication token -* ``VAULT_SECRET_PATH`` — path within the KV engine (e.g. ``path/to/secrets``) Optional environment variables ------------------------------ +* ``VAULT_ADDR`` — Vault server URL (default: ``https://vault.ci.openshift.org``) +* ``VAULT_SECRET_PATH`` — path within the KV engine (default: ``selfservice/psap``) * ``VAULT_KV_MOUNT`` — KV v2 engine mount point (default: ``kv``) * ``FOURNOS_SECRETS_NAMESPACE`` — target Kubernetes namespace (can also - use ``-n``). Defaults to ``psap-secrets``. + use ``-n``). Defaults to ``psap-secrets``. Prerequisites ------------- @@ -24,16 +24,21 @@ Examples -------- -Sync all vault entries under the configured path:: +Sync all vault entries under the configured path (using all defaults):: + + export VAULT_TOKEN="s.xxxxx" + python hacks/sync_vault_secrets.py + +Use custom vault, path, and namespace:: - export VAULT_ADDR="https://vault.example.com" export VAULT_TOKEN="s.xxxxx" + export VAULT_ADDR="https://vault.example.com" export VAULT_SECRET_PATH="path/to/secrets" - python hacks/sync_vault_secrets.py -n psap-secrets + python hacks/sync_vault_secrets.py -n my-namespace Preview without touching the cluster:: - python hacks/sync_vault_secrets.py -n psap-secrets --dry-run + python hacks/sync_vault_secrets.py --dry-run """ from __future__ import annotations @@ -46,12 +51,19 @@ import urllib.error import urllib.request +# Set FOURNOS_NAMESPACE if not set (required for fournos imports) +if not os.environ.get("FOURNOS_NAMESPACE"): + os.environ["FOURNOS_NAMESPACE"] = "not used" + from fournos.core.constants import LABEL_VAULT_ENTRY from fournos.settings import settings logger = logging.getLogger(__name__) KV_MOUNT_DEFAULT = "kv" +VAULT_ADDR_DEFAULT = "https://vault.ci.openshift.org" +VAULT_SECRET_PATH_DEFAULT = "selfservice/psap" +NAMESPACE_DEFAULT = "psap-secrets" LABEL_MANAGED_BY = "app.kubernetes.io/managed-by" ANNOTATION_VAULT_ADDR = "fournos.dev/vault-addr" @@ -369,18 +381,18 @@ def sync( def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description="Sync secrets from a HashiCorp Vault to Kubernetes.", - epilog="Set VAULT_ADDR, VAULT_TOKEN, and VAULT_SECRET_PATH in the environment.", + epilog="Set VAULT_TOKEN in the environment. VAULT_ADDR and VAULT_SECRET_PATH have defaults.", ) parser.add_argument( "-n", "--namespace", - default=settings.secrets_namespace, - help="Target Kubernetes namespace (default: $FOURNOS_SECRETS_NAMESPACE or psap-secrets).", + default=settings.secrets_namespace or NAMESPACE_DEFAULT, + help=f"Target Kubernetes namespace (default: $FOURNOS_SECRETS_NAMESPACE or {NAMESPACE_DEFAULT}).", ) parser.add_argument( "--vault-addr", - default=os.environ.get("VAULT_ADDR", ""), - help="Vault server URL (default: $VAULT_ADDR).", + default=os.environ.get("VAULT_ADDR", VAULT_ADDR_DEFAULT), + help=f"Vault server URL (default: $VAULT_ADDR or {VAULT_ADDR_DEFAULT}).", ) parser.add_argument( "--kv-mount", @@ -389,8 +401,8 @@ def main(argv: list[str] | None = None) -> int: ) parser.add_argument( "--secret-path", - default=os.environ.get("VAULT_SECRET_PATH", ""), - help="Path within the KV engine (default: $VAULT_SECRET_PATH).", + default=os.environ.get("VAULT_SECRET_PATH", VAULT_SECRET_PATH_DEFAULT), + help=f"Path within the KV engine (default: $VAULT_SECRET_PATH or {VAULT_SECRET_PATH_DEFAULT}).", ) parser.add_argument( "--dry-run", @@ -418,20 +430,6 @@ def main(argv: list[str] | None = None) -> int: logger.error("VAULT_TOKEN environment variable is not set") return 1 - if not args.vault_addr: - logger.error("VAULT_ADDR environment variable is not set (or use --vault-addr)") - return 1 - - if not args.secret_path: - logger.error( - "VAULT_SECRET_PATH environment variable is not set (or use --secret-path)" - ) - return 1 - - if not args.namespace: - logger.error("--namespace is required (or set FOURNOS_SECRETS_NAMESPACE)") - return 1 - return sync( vault_addr=args.vault_addr, vault_token=vault_token, From 32e7ba1970f6070037bd393190971a9aae7b9e45 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 28 Apr 2026 12:36:16 +0200 Subject: [PATCH 3/3] hacks: sync_vault_secrets: follow coderabbit suggestions --- hacks/sync_vault_secrets.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/hacks/sync_vault_secrets.py b/hacks/sync_vault_secrets.py index a8094c2..00415cc 100755 --- a/hacks/sync_vault_secrets.py +++ b/hacks/sync_vault_secrets.py @@ -190,7 +190,7 @@ def _list_managed_secrets(v1, namespace: str) -> list[str]: return [secret.metadata.name for secret in secrets.items] except ApiException as exc: logger.error("Failed to list managed secrets: %s", exc) - return [] + raise def _delete_secret(v1, name: str, namespace: str): @@ -203,6 +203,7 @@ def _delete_secret(v1, name: str, namespace: str): except ApiException as exc: if exc.status != 404: logger.error("Failed to delete Secret %s/%s: %s", namespace, name, exc) + raise # --------------------------------------------------------------------------- @@ -249,16 +250,15 @@ def sync( logger.info("Found %d vault entries: %s", len(names), ", ".join(names)) - v1 = None if dry_run else _k8s_core_api() + # Always create v1 client for read operations (needed for destructive sync preview) + v1 = _k8s_core_api() # Track which secrets we process for destructive sync processed_secrets = set() - # Get list of existing managed secrets for destructive sync - existing_managed_secrets = set() - if not dry_run: - existing_managed_secrets = set(_list_managed_secrets(v1, namespace)) - logger.info("Found %d existing managed secrets", len(existing_managed_secrets)) + # Get list of existing managed secrets for destructive sync (always needed for preview) + existing_managed_secrets = set(_list_managed_secrets(v1, namespace)) + logger.info("Found %d existing managed secrets", len(existing_managed_secrets)) errors = 0 skipped_count = 0 @@ -293,6 +293,7 @@ def sync( "Skipping vault %s (contains %s key)", vault_name, SKIP_SYNC_KEY ) skipped_count += 1 + processed_secrets.add(secret_name) continue safe_data: dict[str, str] = {}