Skip to content

Update the secret sync #68

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kpouget merged 3 commits into from
Apr 28, 2026
Merged

Update the secret sync #68

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5637f6a
Enhance vault sync with skip filter and destructive sync
kpouget Apr 28, 2026
683bf8a
hacks: sync_vault_secrets: set PSAP sync default values
kpouget Apr 28, 2026
32e7ba1
hacks: sync_vault_secrets: follow coderabbit suggestions
kpouget Apr 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 107 additions & 29 deletions hacks/sync_vault_secrets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,15 @@

Required environment variables
------------------------------
* ``VAULT_ADDR`` — Vault server URL (e.g. ``https://vault.example.com``)
* ``VAULT_TOKEN`` — Vault authentication token
* ``VAULT_SECRET_PATH`` — path within the KV engine (e.g. ``path/to/secrets``)

Optional environment variables
------------------------------
* ``VAULT_ADDR`` — Vault server URL (default: ``https://vault.ci.openshift.org``)
* ``VAULT_SECRET_PATH`` — path within the KV engine (default: ``selfservice/psap``)
* ``VAULT_KV_MOUNT`` — KV v2 engine mount point (default: ``kv``)
* ``FOURNOS_SECRETS_NAMESPACE`` — target Kubernetes namespace (can also
use ``-n``). Defaults to ``psap-secrets``.
use ``-n``). Defaults to ``psap-secrets``.

Prerequisites
-------------
Expand All @@ -24,16 +24,21 @@

Examples
--------
Sync all vault entries under the configured path::
Sync all vault entries under the configured path (using all defaults)::

export VAULT_TOKEN="s.xxxxx"
python hacks/sync_vault_secrets.py

Use custom vault, path, and namespace::

export VAULT_ADDR="https://vault.example.com"
export VAULT_TOKEN="s.xxxxx"
export VAULT_ADDR="https://vault.example.com"
export VAULT_SECRET_PATH="path/to/secrets"
python hacks/sync_vault_secrets.py -n psap-secrets
python hacks/sync_vault_secrets.py -n my-namespace

Preview without touching the cluster::

python hacks/sync_vault_secrets.py -n psap-secrets --dry-run
python hacks/sync_vault_secrets.py --dry-run
"""

from __future__ import annotations
Expand All @@ -46,17 +51,27 @@
import urllib.error
import urllib.request

# Set FOURNOS_NAMESPACE if not set (required for fournos imports)
if not os.environ.get("FOURNOS_NAMESPACE"):
os.environ["FOURNOS_NAMESPACE"] = "not used"

from fournos.core.constants import LABEL_VAULT_ENTRY
from fournos.settings import settings

logger = logging.getLogger(__name__)

KV_MOUNT_DEFAULT = "kv"
VAULT_ADDR_DEFAULT = "https://vault.ci.openshift.org"
VAULT_SECRET_PATH_DEFAULT = "selfservice/psap"
NAMESPACE_DEFAULT = "psap-secrets"

LABEL_MANAGED_BY = "app.kubernetes.io/managed-by"
ANNOTATION_VAULT_ADDR = "fournos.dev/vault-addr"
ANNOTATION_VAULT_PATH = "fournos.dev/vault-path"
MANAGER_VALUE = "fournos-vault-sync"
SKIP_SYNC_KEY = (
"fournos_skip_sync" # vaults with this key won't be synchronized to the cluster
)


# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -165,6 +180,32 @@ def _apply_secret(
logger.info("Updated Secret %s/%s", namespace, name)


def _list_managed_secrets(v1, namespace: str) -> list[str]:
"""List all secrets managed by fournos-vault-sync."""
from kubernetes.client.exceptions import ApiException

try:
label_selector = f"{LABEL_MANAGED_BY}={MANAGER_VALUE},{LABEL_VAULT_ENTRY}=true"
secrets = v1.list_namespaced_secret(namespace, label_selector=label_selector)
return [secret.metadata.name for secret in secrets.items]
except ApiException as exc:
logger.error("Failed to list managed secrets: %s", exc)
raise


def _delete_secret(v1, name: str, namespace: str):
"""Delete a Kubernetes Secret."""
from kubernetes.client.exceptions import ApiException

try:
v1.delete_namespaced_secret(name, namespace)
logger.info("Deleted Secret %s/%s", namespace, name)
except ApiException as exc:
if exc.status != 404:
logger.error("Failed to delete Secret %s/%s: %s", namespace, name, exc)
raise


# ---------------------------------------------------------------------------
# Core logic
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -206,11 +247,21 @@ def sync(
if not names:
logger.warning("No vault entries found under %s/%s", kv_mount, secret_path)
return 0

logger.info("Found %d vault entries: %s", len(names), ", ".join(names))

v1 = None if dry_run else _k8s_core_api()
# Always create v1 client for read operations (needed for destructive sync preview)
v1 = _k8s_core_api()

# Track which secrets we process for destructive sync
processed_secrets = set()

# Get list of existing managed secrets for destructive sync (always needed for preview)
existing_managed_secrets = set(_list_managed_secrets(v1, namespace))
logger.info("Found %d existing managed secrets", len(existing_managed_secrets))

errors = 0
skipped_count = 0
for vault_name in names:
full_path = f"{secret_path}/{vault_name}"

Expand All @@ -236,6 +287,15 @@ def sync(
logger.warning("Vault entry %s is empty, skipping", full_path)
continue

# Check if vault contains fournos_skip_sync key - if so, skip this vault
if SKIP_SYNC_KEY in kv_data:
logger.info(
"Skipping vault %s (contains %s key)", vault_name, SKIP_SYNC_KEY
)
skipped_count += 1
processed_secrets.add(secret_name)
continue
Comment thread
kpouget marked this conversation as resolved.

safe_data: dict[str, str] = {}
for k, v in kv_data.items():
if _is_vault_metadata(k):
Expand All @@ -260,6 +320,7 @@ def sync(
print(f"[dry-run] Would create/update Secret {namespace}/{secret_name}")
for key in safe_data:
print(f" {key}: <{len(str(safe_data[key]))} chars>")
processed_secrets.add(secret_name)
continue

try:
Expand All @@ -271,6 +332,7 @@ def sync(
vault_addr=vault_addr,
vault_full_path=f"{kv_mount}/{full_path}",
)
processed_secrets.add(secret_name)
except Exception:
logger.exception(
"Failed to apply Secret %s/%s",
Expand All @@ -279,6 +341,36 @@ def sync(
)
errors += 1

# Destructive sync: delete managed secrets that are no longer in vault
secrets_to_delete = existing_managed_secrets - processed_secrets
if secrets_to_delete:
logger.info(
"Found %d managed secrets not in vault, deleting: %s",
len(secrets_to_delete),
", ".join(secrets_to_delete),
)

for secret_name in secrets_to_delete:
if dry_run:
print(f"[dry-run] Would delete Secret {namespace}/{secret_name}")
else:
try:
_delete_secret(v1, secret_name, namespace)
except Exception:
logger.exception(
"Failed to delete Secret %s/%s", namespace, secret_name
)
errors += 1
else:
logger.info("No managed secrets need to be deleted")

if skipped_count > 0:
logger.info(
"Total vault entries skipped (contained %s): %d",
SKIP_SYNC_KEY,
skipped_count,
)

return 1 if errors else 0


Expand All @@ -290,18 +382,18 @@ def sync(
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(
description="Sync secrets from a HashiCorp Vault to Kubernetes.",
epilog="Set VAULT_ADDR, VAULT_TOKEN, and VAULT_SECRET_PATH in the environment.",
epilog="Set VAULT_TOKEN in the environment. VAULT_ADDR and VAULT_SECRET_PATH have defaults.",
)
parser.add_argument(
"-n",
"--namespace",
default=settings.secrets_namespace,
help="Target Kubernetes namespace (default: $FOURNOS_SECRETS_NAMESPACE or psap-secrets).",
default=settings.secrets_namespace or NAMESPACE_DEFAULT,
help=f"Target Kubernetes namespace (default: $FOURNOS_SECRETS_NAMESPACE or {NAMESPACE_DEFAULT}).",
)
parser.add_argument(
"--vault-addr",
default=os.environ.get("VAULT_ADDR", ""),
help="Vault server URL (default: $VAULT_ADDR).",
default=os.environ.get("VAULT_ADDR", VAULT_ADDR_DEFAULT),
help=f"Vault server URL (default: $VAULT_ADDR or {VAULT_ADDR_DEFAULT}).",
)
parser.add_argument(
"--kv-mount",
Expand All @@ -310,8 +402,8 @@ def main(argv: list[str] | None = None) -> int:
)
parser.add_argument(
"--secret-path",
default=os.environ.get("VAULT_SECRET_PATH", ""),
help="Path within the KV engine (default: $VAULT_SECRET_PATH).",
default=os.environ.get("VAULT_SECRET_PATH", VAULT_SECRET_PATH_DEFAULT),
help=f"Path within the KV engine (default: $VAULT_SECRET_PATH or {VAULT_SECRET_PATH_DEFAULT}).",
)
parser.add_argument(
"--dry-run",
Expand Down Expand Up @@ -339,20 +431,6 @@ def main(argv: list[str] | None = None) -> int:
logger.error("VAULT_TOKEN environment variable is not set")
return 1

if not args.vault_addr:
logger.error("VAULT_ADDR environment variable is not set (or use --vault-addr)")
return 1

if not args.secret_path:
logger.error(
"VAULT_SECRET_PATH environment variable is not set (or use --secret-path)"
)
return 1

if not args.namespace:
logger.error("--namespace is required (or set FOURNOS_SECRETS_NAMESPACE)")
return 1

return sync(
vault_addr=args.vault_addr,
vault_token=vault_token,
Expand Down
Loading