diff --git a/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-client-deployment.yaml b/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-client-deployment.yaml new file mode 100644 index 0000000000..78c9d7a358 --- /dev/null +++ b/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-client-deployment.yaml @@ -0,0 +1,26 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: secret-prov-client + labels: + app: secret-prov-client +spec: + template: + metadata: + labels: + app: gsc-ra-tls-secret-prov-client + spec: + volumes: + - name: var-run-aesmd + hostPath: + path: /var/run/aesmd + containers: + - name: gsc-ra-tls-secret-prov-client-container + image: /aks-gsc-secret-prov-client-img + resources: + limits: + kubernetes.azure.com/sgx_epc_mem_in_MiB: 10 + volumeMounts: + - name: var-run-aesmd + mountPath: /var/run/aesmd + restartPolicy: Never diff --git a/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-client.dockerfile b/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-client.dockerfile new file mode 100644 index 0000000000..732e0f0bb2 --- /dev/null +++ b/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-client.dockerfile @@ -0,0 +1,90 @@ +# Steps to create ra-tls-secret-prov min client GSC image for AKS: +# +# STEP 1: Make sure RA-TLS DCAP libraries are built in Gramine via: +# $ cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap +# +# STEP 2: Create base ra-tls-secret-prov min client image +# $ cd gramine +# $ docker build -t \ +# -f Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-client.dockerfile . +# +# STEP 3: Prepare client to connect with remote ra-tls-secret-prov server hosted inside AKS cluster +# 3.1 Provide server dns name as loader.env.SECRET_PROVISION_SERVERS value +# inside Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-client.manifest file. +# +# STEP 4: Create gsc image for ra-tls-secret-prov min client +# $ cd gramine/Tools/gsc +# $ openssl genrsa -3 -out enclave-key.pem 3072 +# $ ./gsc build \ +# images/gramine-aks-attestation/aks-secret-prov-client.manifest +# $ ./gsc sign-image enclave-key.pem +# +# STEP 5: Push resulting image to Docker Hub or your preferred registry +# $ docker tag \ +# / +# $ docker push / +# +# STEP 6: Deploy in AKS confidential compute cluster +# Reference deployment file: +# gsc/images/gramine-aks-attestation/aks-secret-prov-client-deployment.yaml + +FROM ubuntu:18.04 + +RUN apt-get update \ + && env DEBIAN_FRONTEND=noninteractive apt-get install -y wget \ + build-essential \ + gnupg2 \ + libcurl3-gnutls \ + python3 + +# Installing DCAP libraries + +RUN echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' \ + > /etc/apt/sources.list.d/intel-sgx.list \ + && wget https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \ + && apt-key add intel-sgx-deb.key + +RUN apt-get update \ + && apt-get install -y libsgx-urts \ + libsgx-dcap-ql \ + libsgx-quote-ex + +# Build environment of this Dockerfile should point to the root of Gramine directory + +RUN mkdir -p /gramine/Scripts \ + && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/pf_crypt \ + && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/common \ + && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/ra-tls \ + && mkdir -p /gramine/Examples/ra-tls-secret-prov + +# The below files are copied to satisfy Makefile dependencies of gramine/Examples/ra-tls-secret-prov + +COPY Scripts/Makefile.configs /gramine/Scripts/ +COPY Scripts/Makefile.Host /gramine/Scripts/ +COPY Scripts/download /gramine/Scripts/ + +COPY Pal/src/host/Linux-SGX/tools/pf_crypt/pf_crypt /gramine/Pal/src/host/Linux-SGX/tools/pf_crypt/ +COPY Pal/src/host/Linux-SGX/tools/common/libsgx_util.so /gramine/Pal/src/host/Linux-SGX/tools/common/ + +# make sure RA-TLS DCAP libraries are built in host Gramine via: +# cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap + +COPY Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_attest.so /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ +COPY Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_verify_dcap.so /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ +COPY Pal/src/host/Linux-SGX/tools/ra-tls/secret_prov.h /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ + +# If user doesn't want to copy above files, then she can build the ra-tls-secret-prov sample locally +# and copy the entire directory with executables + +COPY Examples/ra-tls-secret-prov /gramine/Examples/ra-tls-secret-prov + +WORKDIR /gramine/Examples/ra-tls-secret-prov + +RUN make clean \ + && make clients dcap + +ENV LD_LIBRARY_PATH = "${LD_LIBRARY_PATH}:./libs" + +ENV PATH = "${PATH}:/gramine/Examples/ra-tls-secret-prov" + +ENTRYPOINT ["/gramine/Examples/ra-tls-secret-prov/secret_prov_min_client"] diff --git a/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-client.manifest b/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-client.manifest new file mode 100644 index 0000000000..04bc1f14d2 --- /dev/null +++ b/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-client.manifest @@ -0,0 +1,15 @@ +# Manifest file for ra-tls-secret-prov min client + +# Secret Provisioning library (client-side) is preloaded +loader.env.LD_PRELOAD = "libs/libsecret_prov_attest.so" + +loader.env.SECRET_PROVISION_SERVERS = ":4433" +loader.env.SECRET_PROVISION_CONSTRUCTOR = "1" +loader.env.SECRET_PROVISION_CA_CHAIN_PATH = "certs/test-ca-sha256.crt" + +# Request remote attestation functionality from Gramine +sgx.remote_attestation = true + +sgx.allowed_files.etchostname = "file:/etc/hostname" +sgx.allowed_files.hosts = "file:/etc/hosts" +sgx.allowed_files.resolv = "file:/etc/resolv.conf" diff --git a/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-server-deployment.yaml b/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-server-deployment.yaml new file mode 100644 index 0000000000..874cae1720 --- /dev/null +++ b/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-server-deployment.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ra-tls-secret-prov-server +spec: + replicas: 1 + selector: + matchLabels: + app: ra-tls-secret-prov-server + template: + metadata: + labels: + app: ra-tls-secret-prov-server + spec: + containers: + - name: ra-tls-secret-prov-server-container + image: /aks-secret-prov-server-img + ports: + - containerPort: 4433 + resources: + limits: + kubernetes.azure.com/sgx_epc_mem_in_MiB: 10 +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + service.beta.kubernetes.io/azure-dns-label-name: + name: ra-tls-secret-prov-server +spec: + type: LoadBalancer + ports: + - port: 4433 + selector: + app: ra-tls-secret-prov-server diff --git a/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-server.dockerfile b/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-server.dockerfile new file mode 100644 index 0000000000..2e90da0b61 --- /dev/null +++ b/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-server.dockerfile @@ -0,0 +1,94 @@ +# Steps to create ra-tls-secret-prov server image for AKS: +# +# STEP 1: Prepare server certificate +# 1.1 Create server certificate signed by your trusted root CA. Ensure Common Name +# field in the server certificate corresponds to used in STEP 5. +# 1.2 Put trusted root CA certificate, server certificate, and server key in +# gramine/Examples/ra-tls-secret-prov/certs directory with existing naming convention. +# +# STEP 2: Make sure RA-TLS DCAP libraries are built in Gramine via: +# $ cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap +# +# STEP 3: Create base ra-tls-secret-prov server image +# $ cd gramine +# $ docker build -t \ +# -f Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-server.dockerfile . +# +# STEP 4: Push resulting image to Docker Hub or your preferred registry +# $ docker tag \ +# / +# $ docker push / +# +# STEP 5: Deploy in AKS confidential compute cluster +# Reference deployment file: +# gsc/images/gramine-aks-attestation/aks-secret-prov-server-deployment.yaml +# +# NOTE: Server can be deployed at a non-confidential compute node as well. However, in that case +# QVE-based dcap verification will fail. + +FROM ubuntu:18.04 + +RUN apt-get update \ + && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ + build-essential \ + gnupg2 \ + libcurl3-gnutls \ + libcurl4-openssl-dev \ + python3 \ + wget + +# Installing Azure DCAP Quote Provider Library (az-dcap-client). +# Here, the version of az-dcap-client should be in sync with the +# az-dcap-client version used for quote generation. +# User can replace the below package with the latest package. + +RUN wget https://github.com/microsoft/Azure-DCAP-Client/releases/download/1.8/az-dcap-client_1.8_amd64_18.04.deb \ + && dpkg -i az-dcap-client_1.8_amd64_18.04.deb + +# Installing DCAP Quote Verification Library +RUN echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' \ + > /etc/apt/sources.list.d/intel-sgx.list \ + && wget https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \ + && apt-key add intel-sgx-deb.key + +RUN apt-get update && apt-get install -y libsgx-dcap-quote-verify + +# Build environment of this Dockerfile should point to the root of Gramine directory + +RUN mkdir -p /gramine/Scripts \ + && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/pf_crypt \ + && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/common \ + && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/ra-tls \ + && mkdir -p /gramine/Examples/ra-tls-secret-prov + +# The below files are copied to satisfy Makefile dependencies of gramine/Examples/ra-tls-secret-prov + +COPY Scripts/Makefile.configs /gramine/Scripts/ +COPY Scripts/Makefile.Host /gramine/Scripts/ +COPY Scripts/download /gramine/Scripts/ + +COPY Pal/src/host/Linux-SGX/tools/pf_crypt/pf_crypt /gramine/Pal/src/host/Linux-SGX/tools/pf_crypt/ +COPY Pal/src/host/Linux-SGX/tools/common/libsgx_util.so /gramine/Pal/src/host/Linux-SGX/tools/common/ + +# make sure RA-TLS DCAP libraries are built in host Gramine via: +# cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap + +COPY Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_attest.so /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ +COPY Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_verify_dcap.so /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ +COPY Pal/src/host/Linux-SGX/tools/ra-tls/secret_prov.h /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ + +# If user doesn't want to copy above files, then she can build the ra-tls-secret-prov sample locally +# and copy the entire directory with executables + +COPY Examples/ra-tls-secret-prov /gramine/Examples/ra-tls-secret-prov + +WORKDIR /gramine/Examples/ra-tls-secret-prov + +RUN make clean \ + && make dcap files/input.txt + +ENV LD_LIBRARY_PATH = "${LD_LIBRARY_PATH}:./libs" + +ENV PATH = "${PATH}:/gramine/Examples/ra-tls-secret-prov" + +ENTRYPOINT ["/gramine/Examples/ra-tls-secret-prov/secret_prov_server_dcap"] diff --git a/Tools/gsc/images/gramine-aks-attestation/gramine_attestation_inside_aks_readme.md b/Tools/gsc/images/gramine-aks-attestation/gramine_attestation_inside_aks_readme.md new file mode 100644 index 0000000000..84ad2bde47 --- /dev/null +++ b/Tools/gsc/images/gramine-aks-attestation/gramine_attestation_inside_aks_readme.md @@ -0,0 +1,85 @@ +# Gramine Attestation Inside AKS cluster + +This guide demonstrates how Gramine DCAP attestation quote can be generated and verified from +within an AKS cluster. Here, we provide an end-to-end example to help Cloud Solution Providers +integrate gramine’s RA-TLS attestation and secret provisioning feature with a confidential compute +cluster managed by Azure Kubernetes Service. The necessary reference wrappers that will enable +gramine to use AKS components such as the AESMD and quote provider libraries are contributed. +A microservice deployment is also provided for the RA-TLS verifier module that can be readily +deployed to the AKS cluster. + +## Create client and server images for gramine attestation samples + +This demonstration is created for ``gramine/Examples/ra-tls-secret-prov`` sample. + +- Steps to create ra-tls-secret-prov server image for AKS: + +```sh +Please refer gramine/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-server.dockerfile +``` + +- Steps to create ra-tls-secret-prov client (min client) gsc image for AKS: + +```sh +Please refer gramine/Tools/gsc/images/gramine-aks-attestation/aks-secret-prov-client.dockerfile +Note: We recommend deploying gsc images on Ubuntu with Linux kernel version 5.11 or higher. +For kernel version lower than 5.11, please uncomment line9 at gsc/templates/apploader.template. +``` + +## Deploy both client and server images inside AKS confidential compute cluster + +AKS confidential compute cluster can be created using following +[link](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-nodes-aks-get-started). + +Gramine performs out-of-proc mode DCAP quote generation. Out-of-proc mode quote generation requires aesmd +service. To fulfill this requirement, AKS provides +[sgxquotehelper daemonset](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-nodes-out-of-proc-attestation). +This feature exposes aesmd service for the container node. The service will internally connect with +az-dcap-client to fetch the platform collateral required for quote generation. In this demo, the +``aks-secret-prov-client-deployment.yaml`` uses aesmd service exposed by AKS with the help of +sgxquotehelper plugin. + +In the ra-tls-secret-prov example, the client will generate out-of-proc mode sgx quote that will be +embedded inside RA-TLS certificate. On receiving the quote, the server will internally verify it +using libsgx-dcap-quote-verify library via az-dcap-client library. Here, +``aks-secret-prov-server-deployment.yaml`` will deploy a ra-tls-secret-prov server container inside + AKS cluster. + +**Deployment**
+ +```sh +$ kubectl apply -f aks-secret-prov-server-deployment.yaml +``` + +Once the server container is in running state, start the client container as shown below + +```sh +$ kubectl apply -f aks-secret-prov-client-deployment.yaml +``` + +At this stage, a successful RA-TLS verification would be completed, and the secrets have been +provisioned from the server to the client container. + +## Steps to verify successful quote generation and quote verification using logs + +Verify the client job is completed + +```sh +$ kubectl get pods +``` +Receive logs to verify the secret has been provisioned to the client + +```sh +$ kubectl logs -l app=gsc-ra-tls-secret-prov-client --tail=50 +``` + +**Expected Output**
+ +--- Received secret = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + +Delete both client and server containers + +```sh +$ kubectl delete -f aks-secret-prov-server-deployment.yaml +$ kubectl delete -f aks-secret-prov-client-deployment.yaml +``` diff --git a/Tools/gsc/templates/apploader.template b/Tools/gsc/templates/apploader.template index 37c8d7e7b8..bbc34295bf 100644 --- a/Tools/gsc/templates/apploader.template +++ b/Tools/gsc/templates/apploader.template @@ -5,6 +5,9 @@ set -ex # Include Meson build output directory in $PYTHONPATH, needed by graphene-sgx-get-token export PYTHONPATH="${PYTHONPATH}:$(find /graphene/meson_build_output/lib -type d -path '*/site-packages')" +# Uncomment below for kernel version lower than 5.11 +# ln -s /dev/sgx/enclave /dev/sgx_enclave + # Set default PAL to Linux-SGX if [ -z "$GSC_PAL" ] || [ "$GSC_PAL" == "Linux-SGX" ] then