use sts endpoint to get minio creds

fridge-job-api/app/main.py

@@ -9,7 +9,6 @@
 from typing import Annotated, Any, Union
 from app.minio_client import MinioClient
 
-
 # Check if running in the Kubernetes cluster
 # If not in the cluster, load environment variables from .env file
 if os.getenv("KUBERNETES_SERVICE_HOST"):
@@ -25,9 +24,6 @@
     FRIDGE_API_ADMIN = os.getenv("FRIDGE_API_ADMIN")
     FRIDGE_API_PASSWORD = os.getenv("FRIDGE_API_PASSWORD")
     ARGO_SERVER = os.getenv("ARGO_SERVER")
-    MINIO_URL = os.getenv("MINIO_URL")
-    MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY")
-    MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY")
 
 # Disable TLS verification in development mode
 VERIFY_TLS = os.getenv("VERIFY_TLS", "False") == "True"
@@ -49,7 +45,7 @@
 
 """
 
-app = FastAPI(title="FRIDGE API", description=description, version="0.2.0")
+app = FastAPI(title="FRIDGE API", description=description, version="0.3.0")
 
 
 # On the Kubernetes cluster, the Argo token is stored in a service account token file on a projected volume
@@ -75,11 +71,16 @@ def argo_token() -> str:
 
 security = HTTPBasic()
 
-# Init minio client (insecure enabled for dev)
+# Init minio client. Will fallback to STS if access/secret key are not set
 minio_client = MinioClient(
     endpoint=os.getenv("MINIO_URL"),
-    access_key=os.getenv("MINIO_ACCESS_KEY"),
-    secret_key=os.getenv("MINIO_SECRET_KEY"),
+    sts_endpoint=os.getenv(
+        "MINIO_STS_URL", "https://sts.minio-operator.svc.cluster.local:4223"
+    ),
+    tenant=os.getenv("MINIO_TENANT_NAME", "argo-artifacts"),
+    access_key=os.getenv("MINIO_ACCESS_KEY", None),
+    secret_key=os.getenv("MINIO_SECRET_KEY", None),
+    secure=os.getenv("MINIO_SECURE", True),
 )
 
 
@@ -372,8 +373,8 @@ async def upload_object(
 async def get_object(
     bucket: str,
     file_name: str,
-    target_file: str = None,
-    version: str = None,
+    target_file: str | None = None,
+    version: str | None = None,
     verified: Annotated[bool, "Verify the request with basic auth"] = Depends(
         verify_request
     ),

fridge-job-api/app/minio_client.py

@@ -1,24 +1,98 @@
-from fastapi import File, UploadFile
+from fastapi import File, UploadFile, HTTPException
 from fastapi.responses import StreamingResponse
 from minio import Minio, versioningconfig, commonconfig
 from io import BytesIO
 from minio.error import S3Error
+import urllib3
+import ssl
+from pathlib import Path
+import xml.etree.ElementTree as ET
+import os
 
 
 class MinioClient:
-    def __init__(self, endpoint: str, access_key: str, secret_key: str):
+    def __init__(
+        self,
+        endpoint: str,
+        sts_endpoint: str | None = None,
+        tenant: str | None = None,
+        access_key: str | None = None,
+        secret_key: str | None = None,
+        secure: bool = False,
+    ):
+        retry_count = 0
+        st = None  # Default session token to None if not using STS
+
+        # Try STS auth if access or secret key is not defined
+        while (access_key is None or secret_key is None) and retry_count < 5:
+            print("Attempting Minio authentication with STS")
+            retry_count = retry_count + 1
+            try:
+                access_key, secret_key, st = self.handle_sts_auth(sts_endpoint, tenant)
+            except Exception as e:
+                print(f"Failed to get keys for minio client: {e}")
+
+        # Exit if minio client keys are not available
+        if access_key is None or secret_key is None:
+            print("Failed to initialise Minio client")
+            exit(1)
+
         self.client = Minio(
-            endpoint, access_key=access_key, secret_key=secret_key, secure=False
+            endpoint,
+            access_key=access_key,
+            secret_key=secret_key,
+            session_token=st,
+            secure=secure,
+        )
+        print("Successfully configured Minio client")
+
+    def handle_sts_auth(self, sts_endpoint, tenant):
+        # Mounted in from the service account to include sts.min.io audience
+        SA_TOKEN_FILE = os.getenv("MINIO_SA_TOKEN_PATH", "/minio/token")
+
+        # Kube CA cert path added by mounted service account, needed for TLS with Minio STS
+        KUBE_CA_CRT = os.getenv(
+            "STS_CA_CERT_FILE", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+        )
+        # Set the environment variable for minio client to use the k8s certificate
+        os.environ["SSL_CERT_FILE"] = KUBE_CA_CRT
+
+        # Read service account token
+        sa_token = Path(SA_TOKEN_FILE).read_text().strip()
+
+        ssl_context = ssl.create_default_context(cafile=KUBE_CA_CRT)
+
+        # Create urllib3 client which accepts kube CA cert
+        http = urllib3.PoolManager(ssl_context=ssl_context)
+
+        # Send the token to the MinIO STS endpoint
+        response = http.request(
+            "POST",
+            f"{sts_endpoint}/sts/{tenant}?Action=AssumeRoleWithWebIdentity&Version=2011-06-15&WebIdentityToken={sa_token}",
         )
 
+        if response.status != 200:
+            print(f"STS request failed: {response.status} {response.data.decode()}")
+            return None, None, None
+        else:
+            root = ET.fromstring(response.data)
+            ns = {"sts": "https://sts.amazonaws.com/doc/2011-06-15/"}
+            credentials = root.find(".//sts:Credentials", ns)
+            access_key = credentials.find("sts:AccessKeyId", ns).text
+            secret_key = credentials.find("sts:SecretAccessKey", ns).text
+            session_token = credentials.find("sts:SessionToken", ns).text
+
+            return access_key, secret_key, session_token
+
     def handle_minio_error(self, error: S3Error):
-        status = 500
         if error._code in ["NoSuchBucket", "NoSuchKey"]:
             status = 404
-        return {"response": error._message, "error": error, "status": status}
+        elif error._code in ["AccessDenied"]:
+            status = 403
+        else:
+            status = 500
 
-    def handle_500_error(self, msg=""):
-        return {"status": 500, "response": f"Unexpected error: {msg}"}
+        raise HTTPException(status_code=status, detail=error.message)
 
     def create_bucket(self, name, enable_versioning=False):
         try:
@@ -30,9 +104,9 @@ def create_bucket(self, name, enable_versioning=False):
                     name, versioningconfig.VersioningConfig(commonconfig.ENABLED)
                 )
         except S3Error as error:
-            return self.handle_minio_error(error)
+            self.handle_minio_error(error)
         except ValueError as error:
-            return self.handle_500_error("Unable to create bucket")
+            raise HTTPException(status_code=500, detail="Unable to create bucket")
 
         return {"response": name, "status": 201}
 
@@ -47,9 +121,11 @@ async def put_object(self, bucket, file: UploadFile = File(...)):
                 content_type=file.content_type,
             )
         except S3Error as error:
-            return self.handle_minio_error(error)
+            self.handle_minio_error(error)
         except Exception as error:
-            return self.handle_500_error("Unable to upload object")
+            raise HTTPException(
+                status_code=500, detail=f"Unable to upload object: {error}"
+            )
 
         return {
             "status": 201,
@@ -70,9 +146,11 @@ def get_object(self, bucket, file_name, target_file=None, version=None):
                 },
             )
         except S3Error as error:
-            return self.handle_minio_error(error)
+            self.handle_minio_error(error)
         except Exception as error:
-            return self.handle_500_error("Unable to get object from bucket")
+            raise HTTPException(
+                status_code=500, detail=f"Unable to get object from bucket: {error}"
+            )
 
     def check_object_exists(self, bucket, file_name, version=None):
         try:
@@ -91,11 +169,16 @@ def delete_object(self, bucket, file_name, version=None):
             if self.check_object_exists(bucket, file_name, version):
                 self.client.remove_object(bucket, file_name, version_id=version)
             else:
-                # Use this path if stat_object result could not be determined
-                return {"status": 500, "response": "Object not deleted"}
+                return {
+                    "status": 404,
+                    "response": f"{file_name} not found in {bucket}",
+                    "version": version,
+                }
         except S3Error as error:
-            return self.handle_minio_error(error)
+            self.handle_minio_error(error)
         except Exception as error:
-            return self.handle_500_error("Unable to delete object from bucket")
+            raise HTTPException(
+                status_code=500, detail="Unable to delete object from bucket"
+            )
 
         return {"status": 200, "response": file_name, "version": version}

infra/fridge/__main__.py

@@ -189,8 +189,7 @@ def patch_namespace(name: str, pss: PodSecurityStandard) -> NamespacePatch:
         fridge_api_admin=config.require_secret("fridge_api_admin"),
         fridge_api_password=config.require_secret("fridge_api_password"),
         minio_url=minio.minio_cluster_url,
-        minio_access_key=config.require_secret("minio_root_user"),
-        minio_secret_key=config.require_secret("minio_root_password"),
+        minio_tenant_name=minio.minio_tenant_name,
         verify_tls=tls_environment is TlsEnvironment.PRODUCTION,
     ),
     opts=ResourceOptions(

infra/fridge/components/api_server.py

@@ -1,5 +1,6 @@
 from pulumi import ComponentResource, ResourceOptions
 from pulumi_kubernetes.apps.v1 import Deployment, DeploymentSpecArgs
+from pulumi_kubernetes.apiextensions import CustomResource
 from pulumi_kubernetes.core.v1 import (
     CapabilitiesArgs,
     ContainerArgs,
@@ -41,17 +42,15 @@ def __init__(
         fridge_api_admin: str,
         fridge_api_password: str,
         minio_url: str,
-        minio_access_key: str,
-        minio_secret_key: str,
+        minio_tenant_name: str,
         verify_tls: bool = True,
     ) -> None:
         self.argo_server_ns = argo_server_ns
         self.argo_workflows_ns = argo_workflows_ns
         self.fridge_api_admin = fridge_api_admin
         self.fridge_api_password = fridge_api_password
         self.minio_url = minio_url
-        self.minio_access_key = minio_access_key
-        self.minio_secret_key = minio_secret_key
+        self.minio_tenant_name = minio_tenant_name
         self.verify_tls = verify_tls
 
 
@@ -121,6 +120,27 @@ def __init__(
             opts=child_opts,
         )
 
+        # Policy binding for the Service account to auth with minio
+        CustomResource(
+            resource_name=f"minio-policy-readwrite",
+            api_version="sts.min.io/v1alpha1",
+            kind="PolicyBinding",
+            metadata=ObjectMetaArgs(
+                name=f"fridge-api-minio-readwrite",
+                namespace=args.minio_tenant_name,
+            ),
+            spec={
+                "application": {
+                    # The namespace that contains the service account for the application
+                    "namespace": fridge_api_sa.metadata.namespace,
+                    # The service account to use for the application
+                    "serviceaccount": fridge_api_sa.metadata.name,
+                },
+                "policies": ["readwrite"],
+            },
+            opts=ResourceOptions(depends_on=[fridge_api_sa]),
+        )
+
         fridge_api_config = Secret(
             "fridge-api-config",
             metadata=ObjectMetaArgs(
@@ -132,8 +152,7 @@ def __init__(
                 "FRIDGE_API_ADMIN": args.fridge_api_admin,
                 "FRIDGE_API_PASSWORD": args.fridge_api_password,
                 "MINIO_URL": args.minio_url,
-                "MINIO_ACCESS_KEY": args.minio_access_key,
-                "MINIO_SECRET_KEY": args.minio_secret_key,
+                "MINIO_TENANT_NAME": args.minio_tenant_name,
                 "VERIFY_TLS": str(args.verify_tls),
             },
             opts=child_opts,
@@ -175,7 +194,6 @@ def __init__(
                 template=PodTemplateSpecArgs(
                     metadata=ObjectMetaArgs(labels={"app": "fridge-api-server"}),
                     spec=PodSpecArgs(
-                        automount_service_account_token=False,
                         containers=[
                             ContainerArgs(
                                 name="fridge-api-server",
@@ -207,6 +225,11 @@ def __init__(
                                         mount_path="/service-account",
                                         read_only=True,
                                     ),
+                                    VolumeMountArgs(
+                                        name="minio-sa",
+                                        mount_path="/minio",
+                                        read_only=True,
+                                    ),
                                 ],
                             )
                         ],
@@ -220,6 +243,20 @@ def __init__(
                                             service_account_token=ServiceAccountTokenProjectionArgs(
                                                 expiration_seconds=3600,
                                                 path="token",
+                                            ),
+                                        )
+                                    ]
+                                ),
+                            ),
+                            VolumeArgs(
+                                name="minio-sa",
+                                projected=ProjectedVolumeSourceArgs(
+                                    sources=[
+                                        VolumeProjectionArgs(
+                                            service_account_token=ServiceAccountTokenProjectionArgs(
+                                                audience="sts.min.io",
+                                                expiration_seconds=3600,
+                                                path="token",
                                             )
                                         )
                                     ]

infra/fridge/components/minio_config.py

@@ -42,7 +42,10 @@ def __init__(
 
         minio_setup_sh = """
             #!/bin/sh
-            mc --insecure alias set "$MINIO_ALIAS" "$MINIO_URL" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
+            # Copy SA cluster cert to be respected by minio
+            mkdir -p /tmp/.mc/certs/CAs/
+            cp /var/run/secrets/kubernetes.io/serviceaccount/ca.crt /tmp/.mc/certs/CAs/
+            mc alias set "$MINIO_ALIAS" "$MINIO_URL" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
             echo "Configuring ingress and egress buckets with anonymous S3 policies"
             mc anonymous set upload "$MINIO_ALIAS/egress"
             mc anonymous set download "$MINIO_ALIAS/ingress"
@@ -103,7 +106,7 @@ def __init__(
                                     EnvVarArgs(
                                         name="MINIO_URL",
                                         value=Output.concat(
-                                            "http://", args.minio_cluster_url, ":80"
+                                            "https://", args.minio_cluster_url, ":443"
                                         ),
                                     ),
                                     EnvVarArgs(

infra/fridge/components/object_storage.py

@@ -124,7 +124,7 @@ def __init__(
                         {"name": "egress"},
                     ],
                     "certificate": {
-                        "requestAutoCert": "false",
+                        "requestAutoCert": "true",
                     },
                     "configuration": {
                         "name": "argo-artifacts-env-configuration",
@@ -214,7 +214,7 @@ def __init__(
                 tls=[
                     IngressTLSArgs(
                         hosts=[self.minio_fqdn],
-                        secret_name="argo-artifacts-tls",
+                        secret_name="argo-artifacts-ingress-tls",
                     )
                 ],
             ),
@@ -224,6 +224,7 @@ def __init__(
             ),
         )
 
+        self.minio_tenant_name = self.minio_tenant.name
         self.register_outputs(
             {
                 "minio_ingress": self.minio_ingress,