Graphene Attestation Inside AKS

Examples/ra-tls-secret-prov/secret_prov_client.manifest.template

@@ -14,6 +14,8 @@ loader.insecure__use_cmdline_argv = true
 # Request remote attestation functionality from Graphene
 sgx.remote_attestation = true
 
+loader.env.SECRET_PROVISION_SERVERS = "dummyserver:80;localhost:4433;anotherdummy:4433"
+
 # Specify your SPID and linkable/unlinkable attestation policy
 sgx.ra_client_spid = "{{ ra_client_spid }}"
 sgx.ra_client_linkable = {{ ra_client_linkable }}

Examples/ra-tls-secret-prov/src/secret_prov_client.c

@@ -31,8 +31,9 @@ int main(int argc, char** argv) {
 
     if (!is_constructor) {
         /* secret provisioning was not run as part of initialization, run it now */
-        ret = secret_provision_start("dummyserver:80;localhost:4433;anotherdummy:4433",
-                                     "certs/test-ca-sha256.crt", &ctx);
+        char* server = getenv(SECRET_PROVISION_SERVERS);
+        ret = secret_provision_start(server, "certs/test-ca-sha256.crt", &ctx);
+
         if (ret < 0) {
             fprintf(stderr, "[error] secret_provision_start() returned %d\n", ret);
             goto out;

Tools/gsc/images/aks-client-deployment.yaml

@@ -0,0 +1,26 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+   name: gsc-ra-tls-secret-prov-client
+   labels:
+      app: gsc-ra-tls-secret-prov-client
+spec:
+   template:
+    metadata:
+       labels:
+           app: gsc-ra-tls-secret-prov-client
+    spec:
+      volumes:
+      - name: var-run-aesmd
+        hostPath:
+          path: /var/run/aesmd
+      containers:
+      - name: gsc-ra-tls-secret-prov-client-container
+        image: <dockerhubusername>/gsc-ra-tls-secret-prov-client-img
+        resources:
+          limits:
+            kubernetes.azure.com/sgx_epc_mem_in_MiB: 25
+        volumeMounts:
+        - name: var-run-aesmd
+          mountPath: /var/run/aesmd
+      restartPolicy: Never

Tools/gsc/images/aks-ra-tls-secret-prov-server.dockerfile

@@ -0,0 +1,93 @@
+# Steps to create ra-tls-secret-prov-server image for AKS:
+#
+# STEP 1: Prepare Server certificate
+#         1.1 Create server certificate signed by your trusted root CA. Ensure Common Name
+#             field in the server certificate corresponds to <AKS-DNS-NAME> used in STEP 5.
+#         1.2 Put trusted root CA certificate, server certificate, and server key in
+#             graphene/Examples/ra-tls-secret-prov/certs directory with existing naming convention.
+#
+# STEP 2: Make sure RA-TLS DCAP libraries are built in Graphene via:
+#         $ cd graphene/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap
+#
+# STEP 3: Create base ra-tls-secret-prov server image
+#         $ cd graphene
+#         $ docker build -t <aks-ra-tls-secret-prov-server-img> \
+#           -f Tools/gsc/images/aks-ra-tls-secret-prov-server.dockerfile .
+#
+# STEP 4: Push resulting image to Docker Hub or your preferred registry
+#         $ docker tag <aks-ra-tls-secret-prov-server-img> \
+#           <dockerhubusername>/<aks-ra-tls-secret-prov-server-img>
+#         $ docker push <dockerhubusername>/<aks-ra-tls-secret-prov-server-img>
+#
+# STEP 5: Deploy <aks-ra-tls-secret-prov-server-img> in AKS confidential compute cluster
+#         Reference deployment file: graphene/Tools/gsc/images/aks-server-deployment.yaml
+#
+# NOTE:  Server can be deployed at non-confidential compute node as well. However, in that case
+#        QVE-based dcap verification will fail.
+
+FROM ubuntu:18.04
+
+RUN apt-get update \
+    && env DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    build-essential \
+    gnupg2 \
+    libcurl3-gnutls \
+    libcurl4-openssl-dev \
+    python3 \
+    wget
+
+# Installing Azure DCAP Quote Provider Library (az-dcap-client).
+# Here, we are using the deb package that we tested for this demo.
+# User can install the latest az-dcap-client as well.
+
+RUN wget https://github.com/microsoft/Azure-DCAP-Client/releases/download/1.8/az-dcap-client_1.8_amd64_18.04.deb \
+    && dpkg -i az-dcap-client_1.8_amd64_18.04.deb
+
+# Installing DCAP Quote Verification Library
+
+RUN echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' \
+    > /etc/apt/sources.list.d/intel-sgx.list \
+    && wget https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && apt-key add intel-sgx-deb.key
+
+RUN apt-get update && apt-get install -y libsgx-dcap-quote-verify
+
+# Build environment of this Dockerfile should point to the root of Graphene directory
+
+RUN mkdir -p /graphene/Scripts \
+    && mkdir -p /graphene/Pal/src/host/Linux-SGX/tools/pf_crypt \
+    && mkdir -p /graphene/Pal/src/host/Linux-SGX/tools/common \
+    && mkdir -p /graphene/Pal/src/host/Linux-SGX/tools/ra-tls \
+    && mkdir -p /graphene/Examples/ra-tls-secret-prov
+
+# The below files are copied to satisfy Makefile dependencies of graphene/Examples/ra-tls-secret-prov
+
+COPY Scripts/Makefile.configs  /graphene/Scripts/
+COPY Scripts/Makefile.Host  /graphene/Scripts/
+COPY Scripts/download  /graphene/Scripts/
+
+COPY Pal/src/host/Linux-SGX/tools/pf_crypt/pf_crypt /graphene/Pal/src/host/Linux-SGX/tools/pf_crypt/
+COPY Pal/src/host/Linux-SGX/tools/common/libsgx_util.so /graphene/Pal/src/host/Linux-SGX/tools/common/
+
+# make sure RA-TLS DCAP libraries are built in host Graphene via:
+# cd graphene/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap
+
+COPY Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_attest.so /graphene/Pal/src/host/Linux-SGX/tools/ra-tls/
+COPY Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_verify_dcap.so /graphene/Pal/src/host/Linux-SGX/tools/ra-tls/
+COPY Pal/src/host/Linux-SGX/tools/ra-tls/secret_prov.h /graphene/Pal/src/host/Linux-SGX/tools/ra-tls/
+
+# If user doesn't want to copy above files, then she can build the ra-tls-secret-prov sample locally
+# and copy the entire directory with executables
+
+COPY Examples/ra-tls-secret-prov /graphene/Examples/ra-tls-secret-prov
+
+WORKDIR /graphene/Examples/ra-tls-secret-prov
+
+RUN make clean \
+    && make dcap files/input.txt
+
+ENV LD_LIBRARY_PATH = "${LD_LIBRARY_PATH}:./libs"
+
+ENV PATH = "${PATH}:/graphene/Examples/ra-tls-secret-prov"
+
+ENTRYPOINT ["/graphene/Examples/ra-tls-secret-prov/secret_prov_server_dcap"]

Tools/gsc/images/aks-server-deployment.yaml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ra-tls-secret-prov-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ra-tls-secret-prov-server
+  template:
+    metadata:
+      labels:
+        app: ra-tls-secret-prov-server
+    spec:
+      containers:
+      - name: ra-tls-secret-prov-server-container
+        image: <dockerhubusername>/ra-tls-secret-prov-server-img
+        ports:
+        - containerPort: 4433
+        resources:
+          limits:
+            kubernetes.azure.com/sgx_epc_mem_in_MiB: 25
+---
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    service.beta.kubernetes.io/azure-dns-label-name: <AKS-DNS-NAME>
+  name: ra-tls-secret-prov-server
+spec:
+  type: LoadBalancer
+  ports:
+  - port: 4433
+  selector:
+    app: ra-tls-secret-prov-server

Tools/gsc/images/graphene_attestation_inside_aks_readme.md

@@ -0,0 +1,96 @@
+# Graphene Attestation Inside AKS cluster
+
+This guide demonstrates how Graphene DCAP attestation quote can be generated and verified from
+within an AKS cluster. Here, we provide an end to end example to help CSPs integrate graphene’s
+RA-TLS attestation and secret provisioning feature with a confidential compute cluster managed by
+Azure Kubernetes Service. The necessary reference wrappers that will enable graphene to use AKS
+components such as the AESMD and quote provider libraries are contributed. A microservice deployment
+is also provided for the RA-TLS verifier module that can be readily deployed to the AKS cluster.
+
+## Create client and server images for graphene attestation samples
+
+This demonstration is created for ``graphene/Examples/ra-tls-secret-prov`` sample.
+
+- Steps to create ra-tls-secret-prov-server image for AKS:
+
+```sh
+Please refer graphene/Tools/gsc/images/aks-ra-tls-secret-prov-server.dockerfile
+```
+
+- Steps to create ra-tls-secret-prov-client gsc image for AKS:
+
+```sh
+STEP 1: Prepare client to connect with remote ra-tls-secret-prov server hosted inside AKS cluster
+        1.1 Provide server dns name <AKS-DNS-NAME> as loader.env.SECRET_PROVISION_SERVERS value
+            inside graphene/Tools/gsc/test/ubuntu18.04-ra-tls-secret-prov.manifest file.
+
+STEP 2: Create gsc image for ra-tls-secret-prov client
+        2.1 Gsc image creation steps for ra-tls-secret-prov-client image are described
+            inside graphene/Tools/gsc/test/ubuntu18.04-ra-tls-secret-prov.manifest.
+
+STEP 3: Push resulting image to Docker Hub or your preferred registry
+        $ docker tag <gsc-ra-tls-secret-prov-client-img> \
+          <dockerhubusername>/<aks-gsc-ra-tls-secret-prov-client-img>
+        $ docker push <dockerhubusername>/<aks-gsc-ra-tls-secret-prov-client-img>
+
+STEP 4: Deploy <aks-gsc-ra-tls-secret-prov-client-img> in confidential compute AKS cluster
+        Reference deployment file: graphene/Tools/gsc/images/aks-client-deployment.yaml
+```
+
+## Deploy both client and server images inside AKS confidential compute cluster
+
+AKS confidential compute cluster can be created using following
+[link](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-nodes-aks-get-started).
+
+Graphene performs out-of-proc mode DCAP quote generation. Out-of-proc mode quote generation requires aesmd
+service. To fulfill this requirement, AKS provides
+[sgxquotehelper daemonset](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-nodes-out-of-proc-attestation).
+This feature exposes aesmd service for the container node. The service will internally connect with
+az-dcap-client to fetch the platform collateral required for quote generation. In this demo, the
+``aks-client-deployment.yaml`` uses aesmd service exposed by AKS with the help of sgxquotehelper
+plugin.
+
+In the ra-tls-secret-prov example, the client will generate out-of-proc mode sgx quote that will be
+embedded inside RA-TLS certificate. On receiving the quote, the server will internally verify it
+using libsgx-dcap-quote-verify library via az-dcap-client library. Here,
+``aks-server-deployment.yaml`` will deploy a ra-tls-secret-prov server container inside AKS cluster.
+
+**Deployment**<br>
+
+```sh
+$ kubectl apply -f aks-server-deployment.yaml
+```
+
+Once the server container is in running state, start the client container as shown below
+
+```sh
+$ kubectl apply -f aks-client-deployment.yaml
+```
+
+At this stage, a successful RA-TLS verification would be completed, and the secrets have been
+provisioned from the server to the client container.
+
+## Steps to verify successful quote generation and quote verification using logs
+
+Verify the client job is completed
+
+```sh
+$ kubectl get jobs -l app=gsc-ra-tls-secret-prov-client
+```
+
+Receive logs to verify the secret has been provisioned to the client
+
+```sh
+$ kubectl logs -l app=gsc-ra-tls-secret-prov-client --tail=50
+```
+
+**Expected Output**<br>
+
+--- Received secret1 = 'XXXXXXXXXXXXXXXXXXXXXXXXXXX', secret2 = 'XX'
+
+Delete both client and server containers
+
+```sh
+$ kubectl apply -f aks-server-deployment.yaml
+$ kubectl apply -f aks-client-deployment.yaml
+```

Tools/gsc/test/ubuntu18.04-ra-tls-secret-prov.manifest

@@ -28,12 +28,14 @@
 # Secret Provisioning library (client-side) is preloaded
 loader.env.LD_PRELOAD = "libs/libsecret_prov_attest.so"
 
+loader.env.SECRET_PROVISION_SERVERS = "dummyserver:80;localhost:4433;anotherdummy:4433"
+
 # Uncomment below lines for secret_prov_min_client and secret_prov_pf_client
 #
 #loader.env.SECRET_PROVISION_CONSTRUCTOR = "1"
 #loader.env.SECRET_PROVISION_SET_PF_KEY = "1"
 #loader.env.SECRET_PROVISION_CA_CHAIN_PATH = "certs/test-ca-sha256.crt"
-#loader.env.SECRET_PROVISION_SERVERS = "dummyserver:80;localhost:4433;anotherdummy:4433"
+
 
 # Request remote attestation functionality from Graphene
 sgx.remote_attestation = true