From 30e27e9ed7e2cc5d16d647a60d3b473476924aa4 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Tue, 14 Sep 2021 11:20:20 +0530 Subject: [PATCH 01/27] Gramine Attestation Inside AKS Signed-off-by: Veena Saini --- .../aks-secret-prov-client-deployment.yaml | 26 +++++ .../aks-secret-prov-client.dockerfile | 90 ++++++++++++++++++ .../aks-secret-prov-client.manifest | 15 +++ .../aks-secret-prov-server-deployment.yaml | 35 +++++++ .../aks-secret-prov-server.dockerfile | 94 +++++++++++++++++++ .../gramine_attestation_inside_aks_readme.md | 86 +++++++++++++++++ templates/apploader.template | 3 + 7 files changed, 349 insertions(+) create mode 100644 examples/gramine-aks-attestation/aks-secret-prov-client-deployment.yaml create mode 100644 examples/gramine-aks-attestation/aks-secret-prov-client.dockerfile create mode 100644 examples/gramine-aks-attestation/aks-secret-prov-client.manifest create mode 100644 examples/gramine-aks-attestation/aks-secret-prov-server-deployment.yaml create mode 100644 examples/gramine-aks-attestation/aks-secret-prov-server.dockerfile create mode 100644 examples/gramine-aks-attestation/gramine_attestation_inside_aks_readme.md diff --git a/examples/gramine-aks-attestation/aks-secret-prov-client-deployment.yaml b/examples/gramine-aks-attestation/aks-secret-prov-client-deployment.yaml new file mode 100644 index 00000000..78c9d7a3 --- /dev/null +++ b/examples/gramine-aks-attestation/aks-secret-prov-client-deployment.yaml @@ -0,0 +1,26 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: secret-prov-client + labels: + app: secret-prov-client +spec: + template: + metadata: + labels: + app: gsc-ra-tls-secret-prov-client + spec: + volumes: + - name: var-run-aesmd + hostPath: + path: /var/run/aesmd + containers: + - name: gsc-ra-tls-secret-prov-client-container + image: /aks-gsc-secret-prov-client-img + resources: + limits: + kubernetes.azure.com/sgx_epc_mem_in_MiB: 10 + volumeMounts: + - name: var-run-aesmd + mountPath: /var/run/aesmd + restartPolicy: Never diff --git a/examples/gramine-aks-attestation/aks-secret-prov-client.dockerfile b/examples/gramine-aks-attestation/aks-secret-prov-client.dockerfile new file mode 100644 index 00000000..d0226a3b --- /dev/null +++ b/examples/gramine-aks-attestation/aks-secret-prov-client.dockerfile @@ -0,0 +1,90 @@ +# Steps to create ra-tls-secret-prov min client GSC image for AKS: +# +# STEP 1: Make sure RA-TLS DCAP libraries are built in Gramine via: +# $ cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap +# +# STEP 2: Create base ra-tls-secret-prov min client image +# $ cd gramine +# $ docker build -t \ +# -f /examples/gramine-aks-attestation/aks-secret-prov-client.dockerfile . +# +# STEP 3: Prepare client to connect with remote ra-tls-secret-prov server hosted inside AKS cluster +# 3.1 Provide server dns name as loader.env.SECRET_PROVISION_SERVERS value +# inside gsc/examples/gramine-aks-attestation/aks-secret-prov-client.manifest file. +# +# STEP 4: Create gsc image for ra-tls-secret-prov min client +# $ cd gsc +# $ openssl genrsa -3 -out enclave-key.pem 3072 +# $ ./gsc build \ +# examples/gramine-aks-attestation/aks-secret-prov-client.manifest +# $ ./gsc sign-image enclave-key.pem +# +# STEP 5: Push resulting image to Docker Hub or your preferred registry +# $ docker tag \ +# / +# $ docker push / +# +# STEP 6: Deploy in AKS confidential compute cluster +# Reference deployment file: +# gsc/examples/gramine-aks-attestation/aks-secret-prov-client-deployment.yaml + +FROM ubuntu:18.04 + +RUN apt-get update \ + && env DEBIAN_FRONTEND=noninteractive apt-get install -y wget \ + build-essential \ + gnupg2 \ + libcurl3-gnutls \ + python3 + +# Installing DCAP libraries + +RUN echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' \ + > /etc/apt/sources.list.d/intel-sgx.list \ + && wget https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \ + && apt-key add intel-sgx-deb.key + +RUN apt-get update \ + && apt-get install -y libsgx-urts \ + libsgx-dcap-ql \ + libsgx-quote-ex + +# Build environment of this Dockerfile should point to the root of Gramine directory + +RUN mkdir -p /gramine/Scripts \ + && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/pf_crypt \ + && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/common \ + && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/ra-tls \ + && mkdir -p /gramine/Examples/ra-tls-secret-prov + +# The below files are copied to satisfy Makefile dependencies of gramine/Examples/ra-tls-secret-prov + +COPY Scripts/Makefile.configs /gramine/Scripts/ +COPY Scripts/Makefile.Host /gramine/Scripts/ +COPY Scripts/download /gramine/Scripts/ + +COPY Pal/src/host/Linux-SGX/tools/pf_crypt/pf_crypt /gramine/Pal/src/host/Linux-SGX/tools/pf_crypt/ +COPY Pal/src/host/Linux-SGX/tools/common/libsgx_util.so /gramine/Pal/src/host/Linux-SGX/tools/common/ + +# make sure RA-TLS DCAP libraries are built in host Gramine via: +# cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap + +COPY Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_attest.so /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ +COPY Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_verify_dcap.so /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ +COPY Pal/src/host/Linux-SGX/tools/ra-tls/secret_prov.h /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ + +# If user doesn't want to copy above files, then she can build the ra-tls-secret-prov sample locally +# and copy the entire directory with executables + +COPY Examples/ra-tls-secret-prov /gramine/Examples/ra-tls-secret-prov + +WORKDIR /gramine/Examples/ra-tls-secret-prov + +RUN make clean \ + && make clients dcap + +ENV LD_LIBRARY_PATH = "${LD_LIBRARY_PATH}:./libs" + +ENV PATH = "${PATH}:/gramine/Examples/ra-tls-secret-prov" + +ENTRYPOINT ["/gramine/Examples/ra-tls-secret-prov/secret_prov_min_client"] diff --git a/examples/gramine-aks-attestation/aks-secret-prov-client.manifest b/examples/gramine-aks-attestation/aks-secret-prov-client.manifest new file mode 100644 index 00000000..04bc1f14 --- /dev/null +++ b/examples/gramine-aks-attestation/aks-secret-prov-client.manifest @@ -0,0 +1,15 @@ +# Manifest file for ra-tls-secret-prov min client + +# Secret Provisioning library (client-side) is preloaded +loader.env.LD_PRELOAD = "libs/libsecret_prov_attest.so" + +loader.env.SECRET_PROVISION_SERVERS = ":4433" +loader.env.SECRET_PROVISION_CONSTRUCTOR = "1" +loader.env.SECRET_PROVISION_CA_CHAIN_PATH = "certs/test-ca-sha256.crt" + +# Request remote attestation functionality from Gramine +sgx.remote_attestation = true + +sgx.allowed_files.etchostname = "file:/etc/hostname" +sgx.allowed_files.hosts = "file:/etc/hosts" +sgx.allowed_files.resolv = "file:/etc/resolv.conf" diff --git a/examples/gramine-aks-attestation/aks-secret-prov-server-deployment.yaml b/examples/gramine-aks-attestation/aks-secret-prov-server-deployment.yaml new file mode 100644 index 00000000..874cae17 --- /dev/null +++ b/examples/gramine-aks-attestation/aks-secret-prov-server-deployment.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ra-tls-secret-prov-server +spec: + replicas: 1 + selector: + matchLabels: + app: ra-tls-secret-prov-server + template: + metadata: + labels: + app: ra-tls-secret-prov-server + spec: + containers: + - name: ra-tls-secret-prov-server-container + image: /aks-secret-prov-server-img + ports: + - containerPort: 4433 + resources: + limits: + kubernetes.azure.com/sgx_epc_mem_in_MiB: 10 +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + service.beta.kubernetes.io/azure-dns-label-name: + name: ra-tls-secret-prov-server +spec: + type: LoadBalancer + ports: + - port: 4433 + selector: + app: ra-tls-secret-prov-server diff --git a/examples/gramine-aks-attestation/aks-secret-prov-server.dockerfile b/examples/gramine-aks-attestation/aks-secret-prov-server.dockerfile new file mode 100644 index 00000000..9416d4df --- /dev/null +++ b/examples/gramine-aks-attestation/aks-secret-prov-server.dockerfile @@ -0,0 +1,94 @@ +# Steps to create ra-tls-secret-prov server image for AKS: +# +# STEP 1: Prepare server certificate +# 1.1 Create server certificate signed by your trusted root CA. Ensure Common Name +# field in the server certificate corresponds to used in STEP 5. +# 1.2 Put trusted root CA certificate, server certificate, and server key in +# gramine/examples/ra-tls-secret-prov/certs directory with existing naming convention. +# +# STEP 2: Make sure RA-TLS DCAP libraries are built in Gramine via: +# $ cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap +# +# STEP 3: Create base ra-tls-secret-prov server image +# $ cd gramine +# $ docker build -t \ +# -f /examples/gramine-aks-attestation/aks-secret-prov-server.dockerfile . +# +# STEP 4: Push resulting image to Docker Hub or your preferred registry +# $ docker tag \ +# / +# $ docker push / +# +# STEP 5: Deploy in AKS confidential compute cluster +# Reference deployment file: +# gsc/examples/gramine-aks-attestation/aks-secret-prov-server-deployment.yaml +# +# NOTE: Server can be deployed at a non-confidential compute node as well. However, in that case +# QVE-based dcap verification will fail. + +FROM ubuntu:18.04 + +RUN apt-get update \ + && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ + build-essential \ + gnupg2 \ + libcurl3-gnutls \ + libcurl4-openssl-dev \ + python3 \ + wget + +# Installing Azure DCAP Quote Provider Library (az-dcap-client). +# Here, the version of az-dcap-client should be in sync with the +# az-dcap-client version used for quote generation. +# User can replace the below package with the latest package. + +RUN wget https://github.com/microsoft/Azure-DCAP-Client/releases/download/1.8/az-dcap-client_1.8_amd64_18.04.deb \ + && dpkg -i az-dcap-client_1.8_amd64_18.04.deb + +# Installing DCAP Quote Verification Library +RUN echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' \ + > /etc/apt/sources.list.d/intel-sgx.list \ + && wget https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \ + && apt-key add intel-sgx-deb.key + +RUN apt-get update && apt-get install -y libsgx-dcap-quote-verify + +# Build environment of this Dockerfile should point to the root of Gramine directory + +RUN mkdir -p /gramine/Scripts \ + && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/pf_crypt \ + && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/common \ + && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/ra-tls \ + && mkdir -p /gramine/Examples/ra-tls-secret-prov + +# The below files are copied to satisfy Makefile dependencies of gramine/Examples/ra-tls-secret-prov + +COPY Scripts/Makefile.configs /gramine/Scripts/ +COPY Scripts/Makefile.Host /gramine/Scripts/ +COPY Scripts/download /gramine/Scripts/ + +COPY Pal/src/host/Linux-SGX/tools/pf_crypt/pf_crypt /gramine/Pal/src/host/Linux-SGX/tools/pf_crypt/ +COPY Pal/src/host/Linux-SGX/tools/common/libsgx_util.so /gramine/Pal/src/host/Linux-SGX/tools/common/ + +# make sure RA-TLS DCAP libraries are built in host Gramine via: +# cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap + +COPY Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_attest.so /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ +COPY Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_verify_dcap.so /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ +COPY Pal/src/host/Linux-SGX/tools/ra-tls/secret_prov.h /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ + +# If user doesn't want to copy above files, then she can build the ra-tls-secret-prov sample locally +# and copy the entire directory with executables + +COPY Examples/ra-tls-secret-prov /gramine/Examples/ra-tls-secret-prov + +WORKDIR /gramine/Examples/ra-tls-secret-prov + +RUN make clean \ + && make dcap files/input.txt + +ENV LD_LIBRARY_PATH = "${LD_LIBRARY_PATH}:./libs" + +ENV PATH = "${PATH}:/gramine/Examples/ra-tls-secret-prov" + +ENTRYPOINT ["/gramine/Examples/ra-tls-secret-prov/secret_prov_server_dcap"] diff --git a/examples/gramine-aks-attestation/gramine_attestation_inside_aks_readme.md b/examples/gramine-aks-attestation/gramine_attestation_inside_aks_readme.md new file mode 100644 index 00000000..8ca4a54b --- /dev/null +++ b/examples/gramine-aks-attestation/gramine_attestation_inside_aks_readme.md @@ -0,0 +1,86 @@ +# Gramine Attestation Inside AKS cluster + +This guide demonstrates how Gramine DCAP attestation quote can be generated and verified from +within an AKS cluster. Here, we provide an end-to-end example to help Cloud Solution Providers +integrate gramine’s RA-TLS attestation and secret provisioning feature with a confidential compute +cluster managed by Azure Kubernetes Service. The necessary reference wrappers that will enable +gramine to use AKS components such as the AESMD and quote provider libraries are contributed. +A microservice deployment is also provided for the RA-TLS verifier module that can be readily +deployed to the AKS cluster. + +## Create client and server images for gramine attestation samples + +This demonstration is created for ``gramine/Examples/ra-tls-secret-prov`` sample. +In order to create the below two images, user needs to download core [gramine repository](https://github.com/gramineproject/gramine). + +- Steps to create ra-tls-secret-prov server image for AKS: + +```sh +Please refer gsc/examples/gramine-aks-attestation/aks-secret-prov-server.dockerfile +``` + +- Steps to create ra-tls-secret-prov client (min client) gsc image for AKS: + +```sh +Please refer gsc/examples/gramine-aks-attestation/aks-secret-prov-client.dockerfile +Note: We recommend deploying gsc images on Ubuntu with Linux kernel version 5.11 or higher. +For kernel version lower than 5.11, please uncomment line9 at gsc/templates/apploader.template. +``` + +## Deploy both client and server images inside AKS confidential compute cluster + +AKS confidential compute cluster can be created using following +[link](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-nodes-aks-get-started). + +Gramine performs out-of-proc mode DCAP quote generation. Out-of-proc mode quote generation requires aesmd +service. To fulfill this requirement, AKS provides +[sgxquotehelper daemonset](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-nodes-out-of-proc-attestation). +This feature exposes aesmd service for the container node. The service will internally connect with +az-dcap-client to fetch the platform collateral required for quote generation. In this demo, the +``aks-secret-prov-client-deployment.yaml`` uses aesmd service exposed by AKS with the help of +sgxquotehelper plugin. + +In the ra-tls-secret-prov example, the client will generate out-of-proc mode sgx quote that will be +embedded inside RA-TLS certificate. On receiving the quote, the server will internally verify it +using libsgx-dcap-quote-verify library via az-dcap-client library. Here, +``aks-secret-prov-server-deployment.yaml`` will deploy a ra-tls-secret-prov server container inside + AKS cluster. + +**Deployment**
+ +```sh +$ kubectl apply -f aks-secret-prov-server-deployment.yaml +``` + +Once the server container is in running state, start the client container as shown below + +```sh +$ kubectl apply -f aks-secret-prov-client-deployment.yaml +``` + +At this stage, a successful RA-TLS verification would be completed, and the secrets have been +provisioned from the server to the client container. + +## Steps to verify successful quote generation and quote verification using logs + +Verify the client job is completed + +```sh +$ kubectl get pods +``` +Receive logs to verify the secret has been provisioned to the client + +```sh +$ kubectl logs -l app=gsc-ra-tls-secret-prov-client --tail=50 +``` + +**Expected Output**
+ +--- Received secret = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + +Delete both client and server containers + +```sh +$ kubectl delete -f aks-secret-prov-server-deployment.yaml +$ kubectl delete -f aks-secret-prov-client-deployment.yaml +``` diff --git a/templates/apploader.template b/templates/apploader.template index 37c8d7e7..bbc34295 100644 --- a/templates/apploader.template +++ b/templates/apploader.template @@ -5,6 +5,9 @@ set -ex # Include Meson build output directory in $PYTHONPATH, needed by graphene-sgx-get-token export PYTHONPATH="${PYTHONPATH}:$(find /graphene/meson_build_output/lib -type d -path '*/site-packages')" +# Uncomment below for kernel version lower than 5.11 +# ln -s /dev/sgx/enclave /dev/sgx_enclave + # Set default PAL to Linux-SGX if [ -z "$GSC_PAL" ] || [ "$GSC_PAL" == "Linux-SGX" ] then From fa27a8ba15cac46608dbc48a4af5669cfecd936c Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Tue, 14 Sep 2021 14:18:53 +0530 Subject: [PATCH 02/27] Renamed gramine-aks-attestation directory to aks-attestation --- .../README.md} | 56 ++++++++++++++++++- .../aks-secret-prov-client-deployment.yaml | 0 .../aks-secret-prov-client.dockerfile | 30 ---------- .../aks-secret-prov-client.manifest | 0 .../aks-secret-prov-server-deployment.yaml | 0 .../aks-secret-prov-server.dockerfile | 28 ---------- 6 files changed, 53 insertions(+), 61 deletions(-) rename examples/{gramine-aks-attestation/gramine_attestation_inside_aks_readme.md => aks-attestation/README.md} (54%) rename examples/{gramine-aks-attestation => aks-attestation}/aks-secret-prov-client-deployment.yaml (100%) rename examples/{gramine-aks-attestation => aks-attestation}/aks-secret-prov-client.dockerfile (59%) rename examples/{gramine-aks-attestation => aks-attestation}/aks-secret-prov-client.manifest (100%) rename examples/{gramine-aks-attestation => aks-attestation}/aks-secret-prov-server-deployment.yaml (100%) rename examples/{gramine-aks-attestation => aks-attestation}/aks-secret-prov-server.dockerfile (65%) diff --git a/examples/gramine-aks-attestation/gramine_attestation_inside_aks_readme.md b/examples/aks-attestation/README.md similarity index 54% rename from examples/gramine-aks-attestation/gramine_attestation_inside_aks_readme.md rename to examples/aks-attestation/README.md index 8ca4a54b..1207c017 100644 --- a/examples/gramine-aks-attestation/gramine_attestation_inside_aks_readme.md +++ b/examples/aks-attestation/README.md @@ -16,14 +16,64 @@ In order to create the below two images, user needs to download core [gramine re - Steps to create ra-tls-secret-prov server image for AKS: ```sh -Please refer gsc/examples/gramine-aks-attestation/aks-secret-prov-server.dockerfile +STEP 1: Prepare server certificate + 1.1 Create server certificate signed by your trusted root CA. Ensure Common Name + field in the server certificate corresponds to used in STEP 5. + 1.2 Put trusted root CA certificate, server certificate, and server key in + gramine/examples/ra-tls-secret-prov/certs directory with existing naming convention. +STEP 2: Make sure RA-TLS DCAP libraries are built in Gramine via: + $ cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap + +STEP 3: Create base ra-tls-secret-prov server image + $ cd gramine + $ docker build -t \ + -f /examples/aks-attestation/aks-secret-prov-server.dockerfile . + +STEP 4: Push resulting image to Docker Hub or your preferred registry + $ docker tag \ + / + $ docker push / + +STEP 5: Deploy in AKS confidential compute cluster + Reference deployment file: + gsc/examples/aks-attestation/aks-secret-prov-server-deployment.yaml + +NOTE: Server can be deployed at a non-confidential compute node as well. However, in that case + QVE-based dcap verification will fail. ``` - Steps to create ra-tls-secret-prov client (min client) gsc image for AKS: ```sh -Please refer gsc/examples/gramine-aks-attestation/aks-secret-prov-client.dockerfile -Note: We recommend deploying gsc images on Ubuntu with Linux kernel version 5.11 or higher. +STEP 1: Make sure RA-TLS DCAP libraries are built in Gramine via: + $ cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap + +STEP 2: Create base ra-tls-secret-prov min client image + $ cd gramine + $ docker build -t \ + -f /examples/aks-attestation/aks-secret-prov-client.dockerfile . + +STEP 3: Prepare client to connect with remote ra-tls-secret-prov server hosted inside AKS cluster + 3.1 Provide server dns name as loader.env.SECRET_PROVISION_SERVERS value + inside gsc/examples/aks-attestation/aks-secret-prov-client.manifest file. + +STEP 4: Create gsc image for ra-tls-secret-prov min client + $ cd gsc + $ openssl genrsa -3 -out enclave-key.pem 3072 + $ ./gsc build \ + examples/aks-attestation/aks-secret-prov-client.manifest + $ ./gsc sign-image enclave-key.pem + +STEP 5: Push resulting image to Docker Hub or your preferred registry + $ docker tag \ + / + $ docker push / + +STEP 6: Deploy in AKS confidential compute cluster + Reference deployment file: + gsc/examples/aks-attestation/aks-secret-prov-client-deployment.yaml + +NOTE: We recommend deploying gsc images on Ubuntu with Linux kernel version 5.11 or higher. For kernel version lower than 5.11, please uncomment line9 at gsc/templates/apploader.template. ``` diff --git a/examples/gramine-aks-attestation/aks-secret-prov-client-deployment.yaml b/examples/aks-attestation/aks-secret-prov-client-deployment.yaml similarity index 100% rename from examples/gramine-aks-attestation/aks-secret-prov-client-deployment.yaml rename to examples/aks-attestation/aks-secret-prov-client-deployment.yaml diff --git a/examples/gramine-aks-attestation/aks-secret-prov-client.dockerfile b/examples/aks-attestation/aks-secret-prov-client.dockerfile similarity index 59% rename from examples/gramine-aks-attestation/aks-secret-prov-client.dockerfile rename to examples/aks-attestation/aks-secret-prov-client.dockerfile index d0226a3b..60bb6ad9 100644 --- a/examples/gramine-aks-attestation/aks-secret-prov-client.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-client.dockerfile @@ -1,33 +1,3 @@ -# Steps to create ra-tls-secret-prov min client GSC image for AKS: -# -# STEP 1: Make sure RA-TLS DCAP libraries are built in Gramine via: -# $ cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap -# -# STEP 2: Create base ra-tls-secret-prov min client image -# $ cd gramine -# $ docker build -t \ -# -f /examples/gramine-aks-attestation/aks-secret-prov-client.dockerfile . -# -# STEP 3: Prepare client to connect with remote ra-tls-secret-prov server hosted inside AKS cluster -# 3.1 Provide server dns name as loader.env.SECRET_PROVISION_SERVERS value -# inside gsc/examples/gramine-aks-attestation/aks-secret-prov-client.manifest file. -# -# STEP 4: Create gsc image for ra-tls-secret-prov min client -# $ cd gsc -# $ openssl genrsa -3 -out enclave-key.pem 3072 -# $ ./gsc build \ -# examples/gramine-aks-attestation/aks-secret-prov-client.manifest -# $ ./gsc sign-image enclave-key.pem -# -# STEP 5: Push resulting image to Docker Hub or your preferred registry -# $ docker tag \ -# / -# $ docker push / -# -# STEP 6: Deploy in AKS confidential compute cluster -# Reference deployment file: -# gsc/examples/gramine-aks-attestation/aks-secret-prov-client-deployment.yaml - FROM ubuntu:18.04 RUN apt-get update \ diff --git a/examples/gramine-aks-attestation/aks-secret-prov-client.manifest b/examples/aks-attestation/aks-secret-prov-client.manifest similarity index 100% rename from examples/gramine-aks-attestation/aks-secret-prov-client.manifest rename to examples/aks-attestation/aks-secret-prov-client.manifest diff --git a/examples/gramine-aks-attestation/aks-secret-prov-server-deployment.yaml b/examples/aks-attestation/aks-secret-prov-server-deployment.yaml similarity index 100% rename from examples/gramine-aks-attestation/aks-secret-prov-server-deployment.yaml rename to examples/aks-attestation/aks-secret-prov-server-deployment.yaml diff --git a/examples/gramine-aks-attestation/aks-secret-prov-server.dockerfile b/examples/aks-attestation/aks-secret-prov-server.dockerfile similarity index 65% rename from examples/gramine-aks-attestation/aks-secret-prov-server.dockerfile rename to examples/aks-attestation/aks-secret-prov-server.dockerfile index 9416d4df..dc3b6199 100644 --- a/examples/gramine-aks-attestation/aks-secret-prov-server.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-server.dockerfile @@ -1,31 +1,3 @@ -# Steps to create ra-tls-secret-prov server image for AKS: -# -# STEP 1: Prepare server certificate -# 1.1 Create server certificate signed by your trusted root CA. Ensure Common Name -# field in the server certificate corresponds to used in STEP 5. -# 1.2 Put trusted root CA certificate, server certificate, and server key in -# gramine/examples/ra-tls-secret-prov/certs directory with existing naming convention. -# -# STEP 2: Make sure RA-TLS DCAP libraries are built in Gramine via: -# $ cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap -# -# STEP 3: Create base ra-tls-secret-prov server image -# $ cd gramine -# $ docker build -t \ -# -f /examples/gramine-aks-attestation/aks-secret-prov-server.dockerfile . -# -# STEP 4: Push resulting image to Docker Hub or your preferred registry -# $ docker tag \ -# / -# $ docker push / -# -# STEP 5: Deploy in AKS confidential compute cluster -# Reference deployment file: -# gsc/examples/gramine-aks-attestation/aks-secret-prov-server-deployment.yaml -# -# NOTE: Server can be deployed at a non-confidential compute node as well. However, in that case -# QVE-based dcap verification will fail. - FROM ubuntu:18.04 RUN apt-get update \ From 9e31a51037d3a4ea53f7e7f6939a0d1b8a2d7081 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Wed, 22 Sep 2021 18:44:57 +0530 Subject: [PATCH 03/27] updated README and dockerfile for new directory hierarchy --- examples/aks-attestation/README.md | 118 ++++++++++-------- .../aks-secret-prov-client.dockerfile | 32 +---- .../aks-secret-prov-server.dockerfile | 32 +---- 3 files changed, 68 insertions(+), 114 deletions(-) diff --git a/examples/aks-attestation/README.md b/examples/aks-attestation/README.md index 1207c017..1031fcb6 100644 --- a/examples/aks-attestation/README.md +++ b/examples/aks-attestation/README.md @@ -10,72 +10,83 @@ deployed to the AKS cluster. ## Create client and server images for gramine attestation samples -This demonstration is created for ``gramine/Examples/ra-tls-secret-prov`` sample. +This demonstration is created for ``gramine/CI-Examples/ra-tls-secret-prov`` sample. In order to create the below two images, user needs to download core [gramine repository](https://github.com/gramineproject/gramine). -- Steps to create ra-tls-secret-prov server image for AKS: +### Steps to create ra-tls-secret-prov server image for AKS -```sh -STEP 1: Prepare server certificate - 1.1 Create server certificate signed by your trusted root CA. Ensure Common Name - field in the server certificate corresponds to used in STEP 5. - 1.2 Put trusted root CA certificate, server certificate, and server key in - gramine/examples/ra-tls-secret-prov/certs directory with existing naming convention. -STEP 2: Make sure RA-TLS DCAP libraries are built in Gramine via: - $ cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap - -STEP 3: Create base ra-tls-secret-prov server image - $ cd gramine - $ docker build -t \ - -f /examples/aks-attestation/aks-secret-prov-server.dockerfile . - -STEP 4: Push resulting image to Docker Hub or your preferred registry - $ docker tag \ - / - $ docker push / - -STEP 5: Deploy in AKS confidential compute cluster - Reference deployment file: +1. Prepare server certificate + - Create server certificate signed by your trusted root CA. Ensure Common Name + field in the server certificate corresponds to `` used in STEP 5. + - Put trusted root CA certificate, server certificate, and server key in + gramine/CI-Examples/ra-tls-secret-prov/certs directory with existing naming convention. + +2. Make sure Gramine is built with -Ddcap=enabled option `meson setup ... -Ddcap=enabled` + +3. Create base ra-tls-secret-prov server image + + ```sh + $ cd gramine + $ docker build -t \ + -f /examples/aks-attestation/aks-secret-prov-server.dockerfile . + + ``` + +4. Push resulting image to Docker Hub or your preferred registry + + ```sh + $ docker tag \ + / + $ docker push / + + ``` +5. Deploy `` in AKS confidential compute cluster + - Reference deployment file: gsc/examples/aks-attestation/aks-secret-prov-server-deployment.yaml NOTE: Server can be deployed at a non-confidential compute node as well. However, in that case QVE-based dcap verification will fail. -``` -- Steps to create ra-tls-secret-prov client (min client) gsc image for AKS: +### Steps to create ra-tls-secret-prov client image for AKS -```sh -STEP 1: Make sure RA-TLS DCAP libraries are built in Gramine via: - $ cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap - -STEP 2: Create base ra-tls-secret-prov min client image - $ cd gramine - $ docker build -t \ - -f /examples/aks-attestation/aks-secret-prov-client.dockerfile . - -STEP 3: Prepare client to connect with remote ra-tls-secret-prov server hosted inside AKS cluster - 3.1 Provide server dns name as loader.env.SECRET_PROVISION_SERVERS value - inside gsc/examples/aks-attestation/aks-secret-prov-client.manifest file. - -STEP 4: Create gsc image for ra-tls-secret-prov min client - $ cd gsc - $ openssl genrsa -3 -out enclave-key.pem 3072 - $ ./gsc build \ - examples/aks-attestation/aks-secret-prov-client.manifest - $ ./gsc sign-image enclave-key.pem - -STEP 5: Push resulting image to Docker Hub or your preferred registry - $ docker tag \ - / - $ docker push / - -STEP 6: Deploy in AKS confidential compute cluster - Reference deployment file: +1. Make sure Gramine is built with -Ddcap=enabled option `meson setup ... -Ddcap=enabled` + +2. Create base ra-tls-secret-prov min client image + + ```sh + $ cd gramine + $ docker build -t \ + -f /examples/aks-attestation/aks-secret-prov-client.dockerfile . + + ``` +3. Prepare client to connect with remote ra-tls-secret-prov server hosted inside AKS cluster + - Provide server dns name `` as `loader.env.SECRET_PROVISION_SERVERS` value + inside gsc/examples/aks-attestation/aks-secret-prov-client.manifest file. + +4. Create gsc image for ra-tls-secret-prov min client + + ```sh + $ cd gsc + $ openssl genrsa -3 -out enclave-key.pem 3072 + $ ./gsc build \ + examples/aks-attestation/aks-secret-prov-client.manifest + $ ./gsc sign-image enclave-key.pem + + ``` +5. Push resulting image to Docker Hub or your preferred registry + + ```sh + $ docker tag \ + / + $ docker push / + + ``` +6. Deploy `` in AKS confidential compute cluster + - Reference deployment file: gsc/examples/aks-attestation/aks-secret-prov-client-deployment.yaml NOTE: We recommend deploying gsc images on Ubuntu with Linux kernel version 5.11 or higher. For kernel version lower than 5.11, please uncomment line9 at gsc/templates/apploader.template. -``` ## Deploy both client and server images inside AKS confidential compute cluster @@ -114,7 +125,6 @@ provisioned from the server to the client container. ## Steps to verify successful quote generation and quote verification using logs Verify the client job is completed - ```sh $ kubectl get pods ``` diff --git a/examples/aks-attestation/aks-secret-prov-client.dockerfile b/examples/aks-attestation/aks-secret-prov-client.dockerfile index 60bb6ad9..de9c0b46 100644 --- a/examples/aks-attestation/aks-secret-prov-client.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-client.dockerfile @@ -21,40 +21,12 @@ RUN apt-get update \ # Build environment of this Dockerfile should point to the root of Gramine directory -RUN mkdir -p /gramine/Scripts \ - && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/pf_crypt \ - && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/common \ - && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/ra-tls \ - && mkdir -p /gramine/Examples/ra-tls-secret-prov +# Before copy do "make clean && make secret_prov_min_client" in CI-Examples/ra-tls-secret-prov directory -# The below files are copied to satisfy Makefile dependencies of gramine/Examples/ra-tls-secret-prov - -COPY Scripts/Makefile.configs /gramine/Scripts/ -COPY Scripts/Makefile.Host /gramine/Scripts/ -COPY Scripts/download /gramine/Scripts/ - -COPY Pal/src/host/Linux-SGX/tools/pf_crypt/pf_crypt /gramine/Pal/src/host/Linux-SGX/tools/pf_crypt/ -COPY Pal/src/host/Linux-SGX/tools/common/libsgx_util.so /gramine/Pal/src/host/Linux-SGX/tools/common/ - -# make sure RA-TLS DCAP libraries are built in host Gramine via: -# cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap - -COPY Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_attest.so /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ -COPY Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_verify_dcap.so /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ -COPY Pal/src/host/Linux-SGX/tools/ra-tls/secret_prov.h /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ - -# If user doesn't want to copy above files, then she can build the ra-tls-secret-prov sample locally -# and copy the entire directory with executables - -COPY Examples/ra-tls-secret-prov /gramine/Examples/ra-tls-secret-prov +COPY CI-Examples/ra-tls-secret-prov /gramine/Examples/ra-tls-secret-prov WORKDIR /gramine/Examples/ra-tls-secret-prov -RUN make clean \ - && make clients dcap - -ENV LD_LIBRARY_PATH = "${LD_LIBRARY_PATH}:./libs" - ENV PATH = "${PATH}:/gramine/Examples/ra-tls-secret-prov" ENTRYPOINT ["/gramine/Examples/ra-tls-secret-prov/secret_prov_min_client"] diff --git a/examples/aks-attestation/aks-secret-prov-server.dockerfile b/examples/aks-attestation/aks-secret-prov-server.dockerfile index dc3b6199..3894cf81 100644 --- a/examples/aks-attestation/aks-secret-prov-server.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-server.dockerfile @@ -27,40 +27,12 @@ RUN apt-get update && apt-get install -y libsgx-dcap-quote-verify # Build environment of this Dockerfile should point to the root of Gramine directory -RUN mkdir -p /gramine/Scripts \ - && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/pf_crypt \ - && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/common \ - && mkdir -p /gramine/Pal/src/host/Linux-SGX/tools/ra-tls \ - && mkdir -p /gramine/Examples/ra-tls-secret-prov +# Before copy do "make clean && make dcap" in CI-Examples/ra-tls-secret-prov directory -# The below files are copied to satisfy Makefile dependencies of gramine/Examples/ra-tls-secret-prov - -COPY Scripts/Makefile.configs /gramine/Scripts/ -COPY Scripts/Makefile.Host /gramine/Scripts/ -COPY Scripts/download /gramine/Scripts/ - -COPY Pal/src/host/Linux-SGX/tools/pf_crypt/pf_crypt /gramine/Pal/src/host/Linux-SGX/tools/pf_crypt/ -COPY Pal/src/host/Linux-SGX/tools/common/libsgx_util.so /gramine/Pal/src/host/Linux-SGX/tools/common/ - -# make sure RA-TLS DCAP libraries are built in host Gramine via: -# cd gramine/Pal/src/host/Linux-SGX/tools/ra-tls && make dcap - -COPY Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_attest.so /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ -COPY Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_verify_dcap.so /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ -COPY Pal/src/host/Linux-SGX/tools/ra-tls/secret_prov.h /gramine/Pal/src/host/Linux-SGX/tools/ra-tls/ - -# If user doesn't want to copy above files, then she can build the ra-tls-secret-prov sample locally -# and copy the entire directory with executables - -COPY Examples/ra-tls-secret-prov /gramine/Examples/ra-tls-secret-prov +COPY CI-Examples/ra-tls-secret-prov /gramine/Examples/ra-tls-secret-prov WORKDIR /gramine/Examples/ra-tls-secret-prov -RUN make clean \ - && make dcap files/input.txt - -ENV LD_LIBRARY_PATH = "${LD_LIBRARY_PATH}:./libs" - ENV PATH = "${PATH}:/gramine/Examples/ra-tls-secret-prov" ENTRYPOINT ["/gramine/Examples/ra-tls-secret-prov/secret_prov_server_dcap"] From c93d61a855ed55b493822e255b06c5fb20431286 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Fri, 24 Sep 2021 12:08:34 +0530 Subject: [PATCH 04/27] Addressed review comments --- examples/aks-attestation/README.md | 78 ++++++++++--------- .../aks-secret-prov-client-deployment.yaml | 2 +- .../aks-secret-prov-client.dockerfile | 10 +-- .../aks-secret-prov-client.manifest | 2 +- .../aks-secret-prov-server-deployment.yaml | 2 +- .../aks-secret-prov-server.dockerfile | 16 ++-- templates/apploader.template | 3 - 7 files changed, 55 insertions(+), 58 deletions(-) diff --git a/examples/aks-attestation/README.md b/examples/aks-attestation/README.md index 1031fcb6..f10895f7 100644 --- a/examples/aks-attestation/README.md +++ b/examples/aks-attestation/README.md @@ -2,68 +2,71 @@ This guide demonstrates how Gramine DCAP attestation quote can be generated and verified from within an AKS cluster. Here, we provide an end-to-end example to help Cloud Solution Providers -integrate gramine’s RA-TLS attestation and secret provisioning feature with a confidential compute +integrate Gramine’s RA-TLS attestation and secret provisioning feature with a confidential compute cluster managed by Azure Kubernetes Service. The necessary reference wrappers that will enable -gramine to use AKS components such as the AESMD and quote provider libraries are contributed. +Gramine to use AKS components such as the AESMD and quote provider libraries are contributed. A microservice deployment is also provided for the RA-TLS verifier module that can be readily deployed to the AKS cluster. -## Create client and server images for gramine attestation samples +## Preparing client and server images This demonstration is created for ``gramine/CI-Examples/ra-tls-secret-prov`` sample. -In order to create the below two images, user needs to download core [gramine repository](https://github.com/gramineproject/gramine). +In order to create the below two images, user needs to download core [Gramine repository](https://github.com/gramineproject/gramine). -### Steps to create ra-tls-secret-prov server image for AKS +### Creating server image -1. Prepare server certificate - - Create server certificate signed by your trusted root CA. Ensure Common Name - field in the server certificate corresponds to `` used in STEP 5. +1. Prepare server certificate: + - Create server certificate signed by your trusted root CA. Ensure "Common Name" + field in the server certificate corresponds to `` used in step 5. - Put trusted root CA certificate, server certificate, and server key in - gramine/CI-Examples/ra-tls-secret-prov/certs directory with existing naming convention. + `gramine/CI-Examples/ra-tls-secret-prov/certs` directory with existing naming convention. -2. Make sure Gramine is built with -Ddcap=enabled option `meson setup ... -Ddcap=enabled` +2. Make sure Gramine is built with `meson setup ... -Ddcap=enabled`. -3. Create base ra-tls-secret-prov server image +3. Create base ra-tls-secret-prov server image: ```sh + $ cd gramine/CI-Examples/ra-tls-secret-prov + $ make clean && make dcap $ cd gramine $ docker build -t \ -f /examples/aks-attestation/aks-secret-prov-server.dockerfile . - ``` -4. Push resulting image to Docker Hub or your preferred registry +4. Push resulting image to Docker Hub or your preferred registry: ```sh $ docker tag \ / $ docker push / - ``` -5. Deploy `` in AKS confidential compute cluster + +5. Deploy `` in AKS confidential compute cluster: - Reference deployment file: - gsc/examples/aks-attestation/aks-secret-prov-server-deployment.yaml + `gsc/examples/aks-attestation/aks-secret-prov-server-deployment.yaml` NOTE: Server can be deployed at a non-confidential compute node as well. However, in that case QVE-based dcap verification will fail. -### Steps to create ra-tls-secret-prov client image for AKS +### Creating client image -1. Make sure Gramine is built with -Ddcap=enabled option `meson setup ... -Ddcap=enabled` +1. Make sure Gramine is built with `meson setup ... -Ddcap=enabled`. -2. Create base ra-tls-secret-prov min client image +2. Create base ra-tls-secret-prov min client image: ```sh + $ cd gramine/CI-Examples/ra-tls-secret-prov + $ make clean && make secret_prov_min_client $ cd gramine $ docker build -t \ -f /examples/aks-attestation/aks-secret-prov-client.dockerfile . - ``` -3. Prepare client to connect with remote ra-tls-secret-prov server hosted inside AKS cluster + +3. Prepare client to connect with remote ra-tls-secret-prov server hosted inside AKS cluster: - Provide server dns name `` as `loader.env.SECRET_PROVISION_SERVERS` value - inside gsc/examples/aks-attestation/aks-secret-prov-client.manifest file. + inside `gsc/examples/aks-attestation/aks-secret-prov-client.manifest` file. -4. Create gsc image for ra-tls-secret-prov min client +4. Create gsc image for ra-tls-secret-prov min client: ```sh $ cd gsc @@ -71,24 +74,23 @@ NOTE: Server can be deployed at a non-confidential compute node as well. Howeve $ ./gsc build \ examples/aks-attestation/aks-secret-prov-client.manifest $ ./gsc sign-image enclave-key.pem - ``` -5. Push resulting image to Docker Hub or your preferred registry + +5. Push resulting image to Docker Hub or your preferred registry: ```sh $ docker tag \ / $ docker push / - ``` -6. Deploy `` in AKS confidential compute cluster + +6. Deploy `` in AKS confidential compute cluster: - Reference deployment file: - gsc/examples/aks-attestation/aks-secret-prov-client-deployment.yaml + `gsc/examples/aks-attestation/aks-secret-prov-client-deployment.yaml` NOTE: We recommend deploying gsc images on Ubuntu with Linux kernel version 5.11 or higher. -For kernel version lower than 5.11, please uncomment line9 at gsc/templates/apploader.template. -## Deploy both client and server images inside AKS confidential compute cluster +## Deploying client and server images inside AKS Confidential Compute cluster AKS confidential compute cluster can be created using following [link](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-nodes-aks-get-started). @@ -107,13 +109,13 @@ using libsgx-dcap-quote-verify library via az-dcap-client library. Here, ``aks-secret-prov-server-deployment.yaml`` will deploy a ra-tls-secret-prov server container inside AKS cluster. -**Deployment**
+### Deployment ```sh $ kubectl apply -f aks-secret-prov-server-deployment.yaml ``` -Once the server container is in running state, start the client container as shown below +Once the server container is in running state, start the client container as shown below: ```sh $ kubectl apply -f aks-secret-prov-client-deployment.yaml @@ -122,21 +124,23 @@ $ kubectl apply -f aks-secret-prov-client-deployment.yaml At this stage, a successful RA-TLS verification would be completed, and the secrets have been provisioned from the server to the client container. -## Steps to verify successful quote generation and quote verification using logs +## Checking SGX quote generation and verification + +Verify the client job is completed: -Verify the client job is completed ```sh $ kubectl get pods ``` -Receive logs to verify the secret has been provisioned to the client + +Receive logs to verify the secret has been provisioned to the client: ```sh $ kubectl logs -l app=gsc-ra-tls-secret-prov-client --tail=50 ``` -**Expected Output**
+### Expected Output ---- Received secret = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' +`--- Received secret = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'` Delete both client and server containers diff --git a/examples/aks-attestation/aks-secret-prov-client-deployment.yaml b/examples/aks-attestation/aks-secret-prov-client-deployment.yaml index 78c9d7a3..f0ce86d5 100644 --- a/examples/aks-attestation/aks-secret-prov-client-deployment.yaml +++ b/examples/aks-attestation/aks-secret-prov-client-deployment.yaml @@ -16,7 +16,7 @@ spec: path: /var/run/aesmd containers: - name: gsc-ra-tls-secret-prov-client-container - image: /aks-gsc-secret-prov-client-img + image: / resources: limits: kubernetes.azure.com/sgx_epc_mem_in_MiB: 10 diff --git a/examples/aks-attestation/aks-secret-prov-client.dockerfile b/examples/aks-attestation/aks-secret-prov-client.dockerfile index de9c0b46..42dbafe8 100644 --- a/examples/aks-attestation/aks-secret-prov-client.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-client.dockerfile @@ -21,12 +21,10 @@ RUN apt-get update \ # Build environment of this Dockerfile should point to the root of Gramine directory -# Before copy do "make clean && make secret_prov_min_client" in CI-Examples/ra-tls-secret-prov directory +RUN mkdir -p /ra-tls-secret-prov -COPY CI-Examples/ra-tls-secret-prov /gramine/Examples/ra-tls-secret-prov +COPY CI-Examples/ra-tls-secret-prov /ra-tls-secret-prov -WORKDIR /gramine/Examples/ra-tls-secret-prov +WORKDIR /ra-tls-secret-prov -ENV PATH = "${PATH}:/gramine/Examples/ra-tls-secret-prov" - -ENTRYPOINT ["/gramine/Examples/ra-tls-secret-prov/secret_prov_min_client"] +ENTRYPOINT ["/ra-tls-secret-prov/secret_prov_min_client"] diff --git a/examples/aks-attestation/aks-secret-prov-client.manifest b/examples/aks-attestation/aks-secret-prov-client.manifest index 04bc1f14..0c3c11d0 100644 --- a/examples/aks-attestation/aks-secret-prov-client.manifest +++ b/examples/aks-attestation/aks-secret-prov-client.manifest @@ -1,7 +1,7 @@ # Manifest file for ra-tls-secret-prov min client # Secret Provisioning library (client-side) is preloaded -loader.env.LD_PRELOAD = "libs/libsecret_prov_attest.so" +loader.env.LD_PRELOAD = "libsecret_prov_attest.so" loader.env.SECRET_PROVISION_SERVERS = ":4433" loader.env.SECRET_PROVISION_CONSTRUCTOR = "1" diff --git a/examples/aks-attestation/aks-secret-prov-server-deployment.yaml b/examples/aks-attestation/aks-secret-prov-server-deployment.yaml index 874cae17..8560a9ff 100644 --- a/examples/aks-attestation/aks-secret-prov-server-deployment.yaml +++ b/examples/aks-attestation/aks-secret-prov-server-deployment.yaml @@ -14,7 +14,7 @@ spec: spec: containers: - name: ra-tls-secret-prov-server-container - image: /aks-secret-prov-server-img + image: / ports: - containerPort: 4433 resources: diff --git a/examples/aks-attestation/aks-secret-prov-server.dockerfile b/examples/aks-attestation/aks-secret-prov-server.dockerfile index 3894cf81..9c4d08b4 100644 --- a/examples/aks-attestation/aks-secret-prov-server.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-server.dockerfile @@ -10,9 +10,9 @@ RUN apt-get update \ wget # Installing Azure DCAP Quote Provider Library (az-dcap-client). -# Here, the version of az-dcap-client should be in sync with the -# az-dcap-client version used for quote generation. -# User can replace the below package with the latest package. +# Here, the version of az-dcap-client should be in sync with the az-dcap-client +# version used for quote generation. User can replace the below package with the +# latest package. RUN wget https://github.com/microsoft/Azure-DCAP-Client/releases/download/1.8/az-dcap-client_1.8_amd64_18.04.deb \ && dpkg -i az-dcap-client_1.8_amd64_18.04.deb @@ -27,12 +27,10 @@ RUN apt-get update && apt-get install -y libsgx-dcap-quote-verify # Build environment of this Dockerfile should point to the root of Gramine directory -# Before copy do "make clean && make dcap" in CI-Examples/ra-tls-secret-prov directory +RUN mkdir -p /ra-tls-secret-prov -COPY CI-Examples/ra-tls-secret-prov /gramine/Examples/ra-tls-secret-prov +COPY CI-Examples/ra-tls-secret-prov /ra-tls-secret-prov -WORKDIR /gramine/Examples/ra-tls-secret-prov +WORKDIR /ra-tls-secret-prov -ENV PATH = "${PATH}:/gramine/Examples/ra-tls-secret-prov" - -ENTRYPOINT ["/gramine/Examples/ra-tls-secret-prov/secret_prov_server_dcap"] +ENTRYPOINT ["/ra-tls-secret-prov/secret_prov_server_dcap"] diff --git a/templates/apploader.template b/templates/apploader.template index bbc34295..37c8d7e7 100644 --- a/templates/apploader.template +++ b/templates/apploader.template @@ -5,9 +5,6 @@ set -ex # Include Meson build output directory in $PYTHONPATH, needed by graphene-sgx-get-token export PYTHONPATH="${PYTHONPATH}:$(find /graphene/meson_build_output/lib -type d -path '*/site-packages')" -# Uncomment below for kernel version lower than 5.11 -# ln -s /dev/sgx/enclave /dev/sgx_enclave - # Set default PAL to Linux-SGX if [ -z "$GSC_PAL" ] || [ "$GSC_PAL" == "Linux-SGX" ] then From 360d710fa79e8f74c5aabe6655354247049d6620 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Fri, 24 Sep 2021 17:02:45 +0530 Subject: [PATCH 05/27] Update Readme --- examples/aks-attestation/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/aks-attestation/README.md b/examples/aks-attestation/README.md index f10895f7..5f8515f5 100644 --- a/examples/aks-attestation/README.md +++ b/examples/aks-attestation/README.md @@ -45,7 +45,7 @@ In order to create the below two images, user needs to download core [Gramine re - Reference deployment file: `gsc/examples/aks-attestation/aks-secret-prov-server-deployment.yaml` -NOTE: Server can be deployed at a non-confidential compute node as well. However, in that case +**NOTE**: Server can be deployed at a non-confidential compute node as well. However, in that case QVE-based dcap verification will fail. ### Creating client image @@ -66,7 +66,7 @@ NOTE: Server can be deployed at a non-confidential compute node as well. Howeve - Provide server dns name `` as `loader.env.SECRET_PROVISION_SERVERS` value inside `gsc/examples/aks-attestation/aks-secret-prov-client.manifest` file. -4. Create gsc image for ra-tls-secret-prov min client: +4. Create GSC image for ra-tls-secret-prov min client: ```sh $ cd gsc @@ -88,7 +88,7 @@ NOTE: Server can be deployed at a non-confidential compute node as well. Howeve - Reference deployment file: `gsc/examples/aks-attestation/aks-secret-prov-client-deployment.yaml` -NOTE: We recommend deploying gsc images on Ubuntu with Linux kernel version 5.11 or higher. +**NOTE**: We recommend deploying GSC images on Ubuntu with Linux kernel version 5.11 or higher. ## Deploying client and server images inside AKS Confidential Compute cluster From 026d93a06c7315a0b1ab249d4301de29ff6312ab Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Fri, 24 Sep 2021 17:58:00 +0530 Subject: [PATCH 06/27] Removed QVE-based comment --- examples/aks-attestation/README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/examples/aks-attestation/README.md b/examples/aks-attestation/README.md index 5f8515f5..2aad2f5f 100644 --- a/examples/aks-attestation/README.md +++ b/examples/aks-attestation/README.md @@ -45,9 +45,6 @@ In order to create the below two images, user needs to download core [Gramine re - Reference deployment file: `gsc/examples/aks-attestation/aks-secret-prov-server-deployment.yaml` -**NOTE**: Server can be deployed at a non-confidential compute node as well. However, in that case - QVE-based dcap verification will fail. - ### Creating client image 1. Make sure Gramine is built with `meson setup ... -Ddcap=enabled`. From 1eae0ee5cab0ab6e611e0bdc7bc308e217e1d6ba Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Mon, 27 Sep 2021 15:06:57 +0530 Subject: [PATCH 07/27] updated Dockerfile and Manifest file for latest gramine changes --- .../aks-secret-prov-client.dockerfile | 2 ++ .../aks-secret-prov-client.manifest | 15 +++++++++++---- .../aks-secret-prov-server.dockerfile | 10 ++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/examples/aks-attestation/aks-secret-prov-client.dockerfile b/examples/aks-attestation/aks-secret-prov-client.dockerfile index 42dbafe8..928e85f1 100644 --- a/examples/aks-attestation/aks-secret-prov-client.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-client.dockerfile @@ -27,4 +27,6 @@ COPY CI-Examples/ra-tls-secret-prov /ra-tls-secret-prov WORKDIR /ra-tls-secret-prov +ENV PATH = "${PATH}:/ra-tls-secret-prov" + ENTRYPOINT ["/ra-tls-secret-prov/secret_prov_min_client"] diff --git a/examples/aks-attestation/aks-secret-prov-client.manifest b/examples/aks-attestation/aks-secret-prov-client.manifest index 0c3c11d0..995ddb86 100644 --- a/examples/aks-attestation/aks-secret-prov-client.manifest +++ b/examples/aks-attestation/aks-secret-prov-client.manifest @@ -3,13 +3,20 @@ # Secret Provisioning library (client-side) is preloaded loader.env.LD_PRELOAD = "libsecret_prov_attest.so" -loader.env.SECRET_PROVISION_SERVERS = ":4433" +loader.env.SECRET_PROVISION_SERVERS = ":4433" loader.env.SECRET_PROVISION_CONSTRUCTOR = "1" loader.env.SECRET_PROVISION_CA_CHAIN_PATH = "certs/test-ca-sha256.crt" # Request remote attestation functionality from Gramine sgx.remote_attestation = true -sgx.allowed_files.etchostname = "file:/etc/hostname" -sgx.allowed_files.hosts = "file:/etc/hosts" -sgx.allowed_files.resolv = "file:/etc/resolv.conf" +sgx.allowed_files = [ + "file:/etc/nsswitch.conf", + "file:/etc/ethers", + "file:/etc/host.conf", + "file:/etc/hosts", + "file:/etc/group", + "file:/etc/passwd", + "file:/etc/gai.conf", + "file:/etc/resolv.conf", +] diff --git a/examples/aks-attestation/aks-secret-prov-server.dockerfile b/examples/aks-attestation/aks-secret-prov-server.dockerfile index 9c4d08b4..d6f839b6 100644 --- a/examples/aks-attestation/aks-secret-prov-server.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-server.dockerfile @@ -31,6 +31,16 @@ RUN mkdir -p /ra-tls-secret-prov COPY CI-Examples/ra-tls-secret-prov /ra-tls-secret-prov +RUN mkdir -p /ra-tls-secret-prov/libs + +COPY build/Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_verify_dcap.so /ra-tls-secret-prov/libs +COPY build/Pal/src/host/Linux-SGX/tools/common/libsgx_util.so /ra-tls-secret-prov/libs +COPY build/subprojects/mbedtls-mbedtls-2.26.0/libmbedcrypto_gramine.so.6 /ra-tls-secret-prov/libs +COPY build/subprojects/mbedtls-mbedtls-2.26.0/libmbedtls_gramine.so.13 /ra-tls-secret-prov/libs +COPY build/subprojects/mbedtls-mbedtls-2.26.0/libmbedx509_gramine.so.1 /ra-tls-secret-prov/libs + +ENV LD_LIBRARY_PATH = "${LD_LIBRARY_PATH}:/ra-tls-secret-prov/libs" + WORKDIR /ra-tls-secret-prov ENTRYPOINT ["/ra-tls-secret-prov/secret_prov_server_dcap"] From f6e00deab42bf285e7963d20907317b22ada09ab Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Mon, 8 Nov 2021 17:04:45 +0530 Subject: [PATCH 08/27] fixup! updated Dockerfile and Manifest file for latest gramine changes Signed-off-by: Veena Saini --- .../aks-secret-prov-client-deployment.yaml | 3 +++ examples/aks-attestation/aks-secret-prov-client.manifest | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/examples/aks-attestation/aks-secret-prov-client-deployment.yaml b/examples/aks-attestation/aks-secret-prov-client-deployment.yaml index f0ce86d5..bba5ecbf 100644 --- a/examples/aks-attestation/aks-secret-prov-client-deployment.yaml +++ b/examples/aks-attestation/aks-secret-prov-client-deployment.yaml @@ -17,6 +17,9 @@ spec: containers: - name: gsc-ra-tls-secret-prov-client-container image: / + env: + - name: SECRET_PROVISION_SERVERS + value: ":4433" resources: limits: kubernetes.azure.com/sgx_epc_mem_in_MiB: 10 diff --git a/examples/aks-attestation/aks-secret-prov-client.manifest b/examples/aks-attestation/aks-secret-prov-client.manifest index 995ddb86..d7b77bf0 100644 --- a/examples/aks-attestation/aks-secret-prov-client.manifest +++ b/examples/aks-attestation/aks-secret-prov-client.manifest @@ -3,7 +3,7 @@ # Secret Provisioning library (client-side) is preloaded loader.env.LD_PRELOAD = "libsecret_prov_attest.so" -loader.env.SECRET_PROVISION_SERVERS = ":4433" +loader.env.SECRET_PROVISION_SERVERS = { passthrough = true } loader.env.SECRET_PROVISION_CONSTRUCTOR = "1" loader.env.SECRET_PROVISION_CA_CHAIN_PATH = "certs/test-ca-sha256.crt" @@ -11,12 +11,12 @@ loader.env.SECRET_PROVISION_CA_CHAIN_PATH = "certs/test-ca-sha256.crt" sgx.remote_attestation = true sgx.allowed_files = [ - "file:/etc/nsswitch.conf", "file:/etc/ethers", + "file:/etc/gai.conf", + "file:/etc/group", "file:/etc/host.conf", "file:/etc/hosts", - "file:/etc/group", + "file:/etc/nsswitch.conf", "file:/etc/passwd", - "file:/etc/gai.conf", "file:/etc/resolv.conf", ] From eb99532b4933a64cd8cc3b8b645f0675d65a8d9a Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Tue, 9 Nov 2021 19:00:43 +0530 Subject: [PATCH 09/27] added base server and client image generation script Signed-off-by: Veena Saini --- examples/aks-attestation/README.md | 62 +++++------ .../aks-secret-prov-client.dockerfile | 10 +- .../aks-secret-prov-server.dockerfile | 20 ++-- .../base-image-generation-script.sh | 61 +++++++++++ examples/aks-attestation/certs/README | 8 ++ .../aks-attestation/certs/server2-sha256.crt | 101 ++++++++++++++++++ examples/aks-attestation/certs/server2.key | 27 +++++ .../aks-attestation/certs/test-ca-sha256.crt | 32 ++++++ 8 files changed, 269 insertions(+), 52 deletions(-) create mode 100755 examples/aks-attestation/base-image-generation-script.sh create mode 100644 examples/aks-attestation/certs/README create mode 100644 examples/aks-attestation/certs/server2-sha256.crt create mode 100644 examples/aks-attestation/certs/server2.key create mode 100644 examples/aks-attestation/certs/test-ca-sha256.crt diff --git a/examples/aks-attestation/README.md b/examples/aks-attestation/README.md index 2aad2f5f..3070c948 100644 --- a/examples/aks-attestation/README.md +++ b/examples/aks-attestation/README.md @@ -10,30 +10,31 @@ deployed to the AKS cluster. ## Preparing client and server images -This demonstration is created for ``gramine/CI-Examples/ra-tls-secret-prov`` sample. -In order to create the below two images, user needs to download core [Gramine repository](https://github.com/gramineproject/gramine). +This demonstration is created for ``gramine/CI-Examples/ra-tls-secret-prov`` sample. The sample +contains client and server applications, where by-default server is running on localhost:4433. Here, +the client sends its SGX quote to the server for verification. After successful quote verification, +the server sends a secret to the client. To run these client and server applications inside AKS +cluster, user needs to prepare two docker images, each for client and server application. Since, now +the server will no longer run on localhost, instead it will run as part of a container inside AKS +cluster, the server container should be assigned a dns name (e.g., ``) for outside +container visibility. The client will send requests to this dns name. Therefore, for demonstration + we updated ``gramine/CI-Examples/ra-tls-secret-prov/certs`` directory certificates by replacing +"Common Name" field in the server certificate (i.e., `server2-sha256.crt`) from `localhost` to +``. + +In order to create base client and server images for AKS environment, user can execute +base-image-generation-script.sh script. Since, both client and server applications will run +inside containers in AKS cluster, and the client wants to send its SGX quote to the server for +verification, therefore the user needs to graminize the client application, so that it can leverage +SGX capabilities from within a container. Hence, the following two steps create base server image +and gsc-client image for AKS cluster. ### Creating server image -1. Prepare server certificate: - - Create server certificate signed by your trusted root CA. Ensure "Common Name" - field in the server certificate corresponds to `` used in step 5. - - Put trusted root CA certificate, server certificate, and server key in - `gramine/CI-Examples/ra-tls-secret-prov/certs` directory with existing naming convention. +1. The base-image-generation-script.sh script will create server image with the name + aks-secret-prov-server-img:latest. -2. Make sure Gramine is built with `meson setup ... -Ddcap=enabled`. - -3. Create base ra-tls-secret-prov server image: - - ```sh - $ cd gramine/CI-Examples/ra-tls-secret-prov - $ make clean && make dcap - $ cd gramine - $ docker build -t \ - -f /examples/aks-attestation/aks-secret-prov-server.dockerfile . - ``` - -4. Push resulting image to Docker Hub or your preferred registry: +2. Push server image to Docker Hub or your preferred registry: ```sh $ docker tag \ @@ -41,29 +42,16 @@ In order to create the below two images, user needs to download core [Gramine re $ docker push / ``` -5. Deploy `` in AKS confidential compute cluster: +3. Deploy `` in AKS confidential compute cluster: - Reference deployment file: `gsc/examples/aks-attestation/aks-secret-prov-server-deployment.yaml` ### Creating client image -1. Make sure Gramine is built with `meson setup ... -Ddcap=enabled`. - -2. Create base ra-tls-secret-prov min client image: - - ```sh - $ cd gramine/CI-Examples/ra-tls-secret-prov - $ make clean && make secret_prov_min_client - $ cd gramine - $ docker build -t \ - -f /examples/aks-attestation/aks-secret-prov-client.dockerfile . - ``` - -3. Prepare client to connect with remote ra-tls-secret-prov server hosted inside AKS cluster: - - Provide server dns name `` as `loader.env.SECRET_PROVISION_SERVERS` value - inside `gsc/examples/aks-attestation/aks-secret-prov-client.manifest` file. +1. The base-image-generation-script.sh script will create client image with the name + aks-secret-prov-client-img:latest. -4. Create GSC image for ra-tls-secret-prov min client: +2. Create GSC image for ra-tls-secret-prov min client: ```sh $ cd gsc diff --git a/examples/aks-attestation/aks-secret-prov-client.dockerfile b/examples/aks-attestation/aks-secret-prov-client.dockerfile index 928e85f1..27d7df30 100644 --- a/examples/aks-attestation/aks-secret-prov-client.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-client.dockerfile @@ -19,14 +19,12 @@ RUN apt-get update \ libsgx-dcap-ql \ libsgx-quote-ex -# Build environment of this Dockerfile should point to the root of Gramine directory - RUN mkdir -p /ra-tls-secret-prov -COPY CI-Examples/ra-tls-secret-prov /ra-tls-secret-prov +COPY gramine/CI-Examples/ra-tls-secret-prov /ra-tls-secret-prov -WORKDIR /ra-tls-secret-prov +COPY gramine/CI-Examples/ra-tls-secret-prov/secret_prov_min_client /usr/local/bin -ENV PATH = "${PATH}:/ra-tls-secret-prov" +WORKDIR /ra-tls-secret-prov -ENTRYPOINT ["/ra-tls-secret-prov/secret_prov_min_client"] +ENTRYPOINT ["secret_prov_min_client"] diff --git a/examples/aks-attestation/aks-secret-prov-server.dockerfile b/examples/aks-attestation/aks-secret-prov-server.dockerfile index d6f839b6..1d2318ef 100644 --- a/examples/aks-attestation/aks-secret-prov-server.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-server.dockerfile @@ -29,18 +29,20 @@ RUN apt-get update && apt-get install -y libsgx-dcap-quote-verify RUN mkdir -p /ra-tls-secret-prov -COPY CI-Examples/ra-tls-secret-prov /ra-tls-secret-prov +COPY gramine/CI-Examples/ra-tls-secret-prov /ra-tls-secret-prov -RUN mkdir -p /ra-tls-secret-prov/libs +COPY gramine/CI-Examples/ra-tls-secret-prov/secret_prov_server_dcap /usr/local/bin -COPY build/Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_verify_dcap.so /ra-tls-secret-prov/libs -COPY build/Pal/src/host/Linux-SGX/tools/common/libsgx_util.so /ra-tls-secret-prov/libs -COPY build/subprojects/mbedtls-mbedtls-2.26.0/libmbedcrypto_gramine.so.6 /ra-tls-secret-prov/libs -COPY build/subprojects/mbedtls-mbedtls-2.26.0/libmbedtls_gramine.so.13 /ra-tls-secret-prov/libs -COPY build/subprojects/mbedtls-mbedtls-2.26.0/libmbedx509_gramine.so.1 /ra-tls-secret-prov/libs +RUN mkdir -p /ra-tls-secret-prov/libs -ENV LD_LIBRARY_PATH = "${LD_LIBRARY_PATH}:/ra-tls-secret-prov/libs" +COPY gramine/build/Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_verify_dcap.so /ra-tls-secret-prov/libs +COPY gramine/build/Pal/src/host/Linux-SGX/tools/common/libsgx_util.so /ra-tls-secret-prov/libs +COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedcrypto_gramine.so.6 /ra-tls-secret-prov/libs +COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedtls_gramine.so.13 /ra-tls-secret-prov/libs +COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedx509_gramine.so.1 /ra-tls-secret-prov/libs WORKDIR /ra-tls-secret-prov -ENTRYPOINT ["/ra-tls-secret-prov/secret_prov_server_dcap"] +ENV LD_LIBRARY_PATH = "${LD_LIBRARY_PATH}:/ra-tls-secret-prov/libs" + +ENTRYPOINT ["secret_prov_server_dcap"] diff --git a/examples/aks-attestation/base-image-generation-script.sh b/examples/aks-attestation/base-image-generation-script.sh new file mode 100755 index 00000000..db94cd9b --- /dev/null +++ b/examples/aks-attestation/base-image-generation-script.sh @@ -0,0 +1,61 @@ +# install Gramine dependencies + +apt-get install -y \ + autoconf \ + bison \ + build-essential \ + coreutils \ + curl \ + gawk \ + git \ + libcurl4-openssl-dev \ + libprotobuf-c-dev \ + linux-headers-generic \ + ninja-build \ + pkg-config \ + protobuf-c-compiler \ + python3 \ + python3-pip \ + python3-protobuf \ + wget + +python3 -B -m pip install 'toml>=0.10' 'meson>=0.55' + +# Download Gramine + +git clone https://github.com/gramineproject/gramine.git + +# Generate Signing Key + +cd gramine/Pal/src/host/Linux-SGX/signer/ +openssl genrsa -3 -out enclave-key.pem 3072 + +# Build Gramine with DCAP enabled mode + +cd ../../../../../ +meson setup build/ --buildtype=release -Ddirect=enabled -Dsgx=enabled -Ddcap=enabled +ninja -C build/ +sudo ninja -C build/ install + +# Copy dummy server certificate with Common Name as " + +cd CI-Examples/ra-tls-secret-prov +mv certs certs_orig +cp -r ../../../certs ./ + +# Create Server image + +make clean && make dcap +cd ../../../ +docker build -f aks-secret-prov-server.dockerfile -t aks-secret-prov-server-img . + +# Create Client image + +cd gramine/CI-Examples/ra-tls-secret-prov +make clean && make secret_prov_min_client +cd ../../../ +docker build -f aks-secret-prov-client.dockerfile -t aks-secret-prov-client-img . + +# Remove Gramine directory + +rm -r gramine/ diff --git a/examples/aks-attestation/certs/README b/examples/aks-attestation/certs/README new file mode 100644 index 00000000..8877aa05 --- /dev/null +++ b/examples/aks-attestation/certs/README @@ -0,0 +1,8 @@ +This directory contains pre-generated example certificates. In particular: + +- `test-ca-sha256.crt` -- RSA SHA256 root CA certificate in PEM format. Loaded + in client (attester) so that client can verify the server's certificate. +- `server2-sha256.crt` -- RSA SHA256 leaf server certificate in PEM format. + Loaded in server (verifier), so it will send it to the client during TLS + handshake. Common Name `ra-tls-server-aks-dns.eastus.cloudapp.azure.com` +- `server2.key` -- RSA private key in PEM format. Loaded in server (verifier). diff --git a/examples/aks-attestation/certs/server2-sha256.crt b/examples/aks-attestation/certs/server2-sha256.crt new file mode 100644 index 00000000..76bc11b7 --- /dev/null +++ b/examples/aks-attestation/certs/server2-sha256.crt @@ -0,0 +1,101 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 4660 (0x1234) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=NL, ST=NL, L=NL, O=PolarSSL, CN=PolarSSL Test CA + Validity + Not Before: Nov 9 05:11:04 2021 GMT + Not After : Nov 9 05:11:04 2022 GMT + Subject: C=NL, ST=NL, O=PolarSSL, CN=ra-tls-server-aks-dns.eastus.cloudapp.azure.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (2048 bit) + Modulus: + 00:a6:86:86:58:40:f6:02:6e:e1:07:70:d6:bb:25: + ae:1c:bc:d3:15:f0:f8:47:85:85:fd:38:85:e9:99: + bc:8f:63:ef:9e:a9:98:cc:ef:81:81:cf:77:2d:f7: + a9:29:54:e5:cf:dc:59:12:ce:c6:aa:a0:8d:d6:93: + 1b:5c:79:68:81:bb:04:d9:e1:af:11:ec:c6:63:b6: + d4:77:b8:31:8e:c4:bc:46:dd:de:7c:0b:1f:95:60: + e0:15:67:af:47:94:df:25:2b:da:ea:29:89:2e:43: + 5f:43:23:4f:47:db:4b:33:16:14:10:0b:5c:7c:80: + 97:6c:c9:23:ef:d2:16:96:1d:f5:d5:3f:78:fa:17: + 8d:22:47:c1:7d:42:66:19:16:a9:bd:66:42:33:77: + 79:87:4a:6e:85:80:b5:5e:89:ad:31:33:ba:c7:b9: + 19:57:fa:80:80:fd:2d:d5:3c:a9:85:3f:67:c5:1b: + 11:d9:43:c2:97:fd:5e:a8:ba:e5:21:7f:75:22:ff: + 53:26:ec:e1:3a:e4:d0:df:8b:71:2b:b6:d1:c0:dc: + bb:8b:97:07:53:7d:74:98:7e:55:36:82:dc:a2:d2: + 5f:ea:22:85:93:93:90:48:59:11:45:94:b2:82:e8: + 38:84:7b:9b:a0:7d:9d:6a:f2:60:7d:0d:55:11:31: + ab:af + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Comment: + OpenSSL Generated Certificate + X509v3 Subject Key Identifier: + A0:EC:6E:8A:20:C5:81:3F:6F:27:09:7B:D6:94:DF:E0:DB:57:67:55 + X509v3 Authority Key Identifier: + keyid:8A:35:A0:64:6B:D1:3F:28:92:55:12:6D:5A:6D:FD:EE:F0:2A:D5:DB + + Signature Algorithm: sha256WithRSAEncryption + 7d:90:5c:df:55:cd:3e:46:4a:75:0e:a3:9a:d3:56:27:15:5a: + 0c:d2:cb:10:4c:b0:df:e1:88:21:cb:23:a2:c3:0f:d0:ea:d3: + b4:41:65:d4:98:95:a6:7e:4d:5e:d2:32:40:eb:63:ab:5b:70: + 43:b8:0c:4d:5d:31:52:a2:f2:04:f9:aa:55:96:45:16:db:e3: + 9c:db:79:96:8f:3a:c8:a5:f2:f6:d0:69:03:3e:d4:df:1f:9d: + 84:a9:81:ef:73:de:04:59:eb:1f:d3:65:24:e4:92:e8:00:2f: + 9e:d6:01:22:9f:9a:57:ff:52:54:42:0b:a5:b3:71:56:53:14: + 5c:ad:c4:99:0e:81:97:c0:88:73:3b:ef:1f:2b:9d:52:43:68: + a7:78:b6:46:c6:92:cf:f3:a8:87:a9:a5:76:0c:a3:9f:b7:2b: + 51:62:d7:b0:2a:58:eb:e3:56:c9:5e:d9:f7:db:60:c1:70:ef: + 29:08:4e:dc:6e:b6:e5:82:60:e5:5d:c4:68:4f:63:90:7f:af: + 77:f1:89:00:0d:73:be:24:aa:0d:13:93:c3:9c:e2:e9:6b:44: + d7:12:f6:32:69:40:0b:9f:c5:15:80:43:aa:9b:1b:32:fa:00: + b6:9a:c0:c8:db:18:0f:2e:02:94:0e:1a:bf:dc:ad:b3:bb:c6: + ab:9c:6e:80:e3:0e:80:a3:f0:b7:b7:e1:fe:a5:56:1c:75:d3: + 3e:33:5a:62:a0:3f:1b:eb:31:71:2f:92:8b:26:1f:c3:9b:84: + cd:d6:8c:29:b6:62:44:f7:7e:bc:4e:19:ad:76:09:fc:dd:c6: + aa:ce:ec:e3:14:bb:df:c5:6e:c9:32:e7:9b:dd:06:ab:33:04: + 30:22:f3:9e:4f:9b:7e:3f:53:48:04:c2:36:f9:bf:87:cb:8c: + bc:c5:b4:e5:33:df:3d:82:aa:21:ae:ac:75:87:0c:22:54:36: + 4c:84:1e:4d:01:ab:c6:86:c0:88:1e:8e:58:c2:4f:42:4e:2e: + 2f:77:0c:ae:5a:c4:b3:0a:a6:6a:57:ea:fe:33:d2:a5:50:21: + a5:d9:cf:09:4a:4f:3f:04:aa:1d:86:22:e2:7b:0a:88:e9:4e: + 22:c6:f3:a9:9f:fd:8c:d1:b0:4e:b1:d5:16:52:60:41:2a:da: + ba:51:12:40:67:20:37:06:0b:bc:8c:5b:09:c4:de:ff:69:8c: + 39:26:21:15:62:d0:aa:62:b8:bc:8f:67:8b:d5:43:a7:c0:8e: + 86:c4:fe:12:77:96:b5:12:b8:f5:fc:39:78:59:66:e1:ff:ff: + 2b:e4:83:ef:8f:0a:80:50:a2:13:6c:a5:dd:eb:2c:26:80:b3: + 87:72:10:7e:24:db:07:ff +-----BEGIN CERTIFICATE----- +MIIEszCCApugAwIBAgICEjQwDQYJKoZIhvcNAQELBQAwVTELMAkGA1UEBhMCTkwx +CzAJBgNVBAgMAk5MMQswCQYDVQQHDAJOTDERMA8GA1UECgwIUG9sYXJTU0wxGTAX +BgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwHhcNMjExMTA5MDUxMTA0WhcNMjIxMTA5 +MDUxMTA0WjBnMQswCQYDVQQGEwJOTDELMAkGA1UECAwCTkwxETAPBgNVBAoMCFBv +bGFyU1NMMTgwNgYDVQQDDC9yYS10bHMtc2VydmVyLWFrcy1kbnMuZWFzdHVzLmNs +b3VkYXBwLmF6dXJlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +AKaGhlhA9gJu4Qdw1rslrhy80xXw+EeFhf04hemZvI9j756pmMzvgYHPdy33qSlU +5c/cWRLOxqqgjdaTG1x5aIG7BNnhrxHsxmO21He4MY7EvEbd3nwLH5Vg4BVnr0eU +3yUr2uopiS5DX0MjT0fbSzMWFBALXHyAl2zJI+/SFpYd9dU/ePoXjSJHwX1CZhkW +qb1mQjN3eYdKboWAtV6JrTEzuse5GVf6gID9LdU8qYU/Z8UbEdlDwpf9Xqi65SF/ +dSL/Uybs4Trk0N+LcSu20cDcu4uXB1N9dJh+VTaC3KLSX+oihZOTkEhZEUWUsoLo +OIR7m6B9nWryYH0NVRExq68CAwEAAaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhC +AQ0EHxYdT3BlblNTTCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFKDs +boogxYE/bycJe9aU3+DbV2dVMB8GA1UdIwQYMBaAFIo1oGRr0T8oklUSbVpt/e7w +KtXbMA0GCSqGSIb3DQEBCwUAA4ICAQB9kFzfVc0+Rkp1DqOa01YnFVoM0ssQTLDf +4YghyyOiww/Q6tO0QWXUmJWmfk1e0jJA62OrW3BDuAxNXTFSovIE+apVlkUW2+Oc +23mWjzrIpfL20GkDPtTfH52EqYHvc94EWesf02Uk5JLoAC+e1gEin5pX/1JUQgul +s3FWUxRcrcSZDoGXwIhzO+8fK51SQ2ineLZGxpLP86iHqaV2DKOftytRYtewKljr +41bJXtn322DBcO8pCE7cbrblgmDlXcRoT2OQf6938YkADXO+JKoNE5PDnOLpa0TX +EvYyaUALn8UVgEOqmxsy+gC2msDI2xgPLgKUDhq/3K2zu8arnG6A4w6Ao/C3t+H+ +pVYcddM+M1pioD8b6zFxL5KLJh/Dm4TN1owptmJE9368Thmtdgn83caqzuzjFLvf +xW7JMueb3QarMwQwIvOeT5t+P1NIBMI2+b+Hy4y8xbTlM989gqohrqx1hwwiVDZM +hB5NAavGhsCIHo5Ywk9CTi4vdwyuWsSzCqZqV+r+M9KlUCGl2c8JSk8/BKodhiLi +ewqI6U4ixvOpn/2M0bBOsdUWUmBBKtq6URJAZyA3Bgu8jFsJxN7/aYw5JiEVYtCq +Yri8j2eL1UOnwI6GxP4Sd5a1Erj1/Dl4WWbh//8r5IPvjwqAUKITbKXd6ywmgLOH +chB+JNsH/w== +-----END CERTIFICATE----- diff --git a/examples/aks-attestation/certs/server2.key b/examples/aks-attestation/certs/server2.key new file mode 100644 index 00000000..45604cc1 --- /dev/null +++ b/examples/aks-attestation/certs/server2.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEApoaGWED2Am7hB3DWuyWuHLzTFfD4R4WF/TiF6Zm8j2PvnqmY +zO+Bgc93LfepKVTlz9xZEs7GqqCN1pMbXHlogbsE2eGvEezGY7bUd7gxjsS8Rt3e +fAsflWDgFWevR5TfJSva6imJLkNfQyNPR9tLMxYUEAtcfICXbMkj79IWlh311T94 ++heNIkfBfUJmGRapvWZCM3d5h0puhYC1XomtMTO6x7kZV/qAgP0t1TyphT9nxRsR +2UPCl/1eqLrlIX91Iv9TJuzhOuTQ34txK7bRwNy7i5cHU310mH5VNoLcotJf6iKF +k5OQSFkRRZSygug4hHuboH2davJgfQ1VETGrrwIDAQABAoIBABWyc3CuTh91CPbq +OeA6gZwW8j3qnr7a4+FhavlL4aihMchveoxBLJOYT4P6v9y1xhe2MxjjPjlfH/v0 +QN1kqDw2/OPSpvfj7NlzKe3LXxGmCo4FPIbXAC97ZtcDInBmMkjqlORrswcHGjNq +N1s6dGrclpKcmFlUe+ZdssP9kJnQabiESxIjpKHgFVYLdYFeXdGxdvOE8p3hOci2 +ftrHK9/fXym+/MFdywxZCfhzpPaJJyJIqH0gbMZDOLv2rBaymf+FEgfEG4LgFas8 +CZf/hM6HkxcmCMVgcP8Oagb0NEnOAauUSmRowfLVd8Ob7ZlvrQKAkIHgH+shQ3hI +h6E+6LECgYEAzuhqasEHILxQpEwBXcoa9rQJHYIf27NSFNlKsal6IkCZCDlWbdQn +t+xbHPDnyRYv4PVYhh/2catlD6JLivxenTNm1fBdKzo/yMDKM6gubqaIo3B5Q8OB +hDlubFCxboPQ1bq05QAK1d8R4b4CmBqPXDQ/KrfFgEn47YmsrxpVs9kCgYEAzglI +bmqCiNV4Qw2sfp15lIPanuRfE9W2tDbj+ND4rafOJGrDuwEIpTMqSTW1hSOY58EL +w5VMa2+bqqOEufNH21HtpWQv2zVU8EbB0KevdP/f5xjGKhysoWOL/q6l1PPbF1Bg +pY/fPW+9xNuskb9XUXhkZ3tsPYeXPwnsLaGFDscCgYA0zYFi7xH3m7dkxnTOz6Yj +8XwmCLBCrDe1VGLIhjzCAwRK4f8ZljvwpyP1ddFhMwVS/qnxyLRm+9uVqtBmjtNC +gPysemP8hSgD/pdbqX74BA34z9pSndEuwk1Y3Rr63i7sbHJ/wowBWqVAm/IjiRuu +5zJ7n1yKydX6INlt8mWOuQKBgGCqgk0rs6q3XQE4vSF6iLxhUIUT6HfJ4TqUNkla +4hG3lUQIw2RNi9zWMJTAmjWpCehR7k8JER2m84YPygnkVmZT/geHBdz3HK7Ow8g9 +pFhkZIxsVy0S3vupVNlc33kWIEjD8VQ8OCT+9C4UUtbeyvzyW0kh4xrLLu7onOKb +0W2pAoGAfZ9UC4FrLfYgXAYz3WzHuVa4Z/r3PXswI5djINFZIAwLKjD7/E3R7BCq +pOag/PuESJu9CNcgYNURtOeIaW7MN80UN7086y0SnAw6OQXtIrSSHMEkTMr7iSXm +/VtC24AL/L7YSd8fRNXxYAJvkwowOllbNI8uKnzZn9JkaWBZajg= +-----END RSA PRIVATE KEY----- diff --git a/examples/aks-attestation/certs/test-ca-sha256.crt b/examples/aks-attestation/certs/test-ca-sha256.crt new file mode 100644 index 00000000..54028f0b --- /dev/null +++ b/examples/aks-attestation/certs/test-ca-sha256.crt @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFeDCCA2CgAwIBAgIBADANBgkqhkiG9w0BAQsFADBVMQswCQYDVQQGEwJOTDEL +MAkGA1UECAwCTkwxCzAJBgNVBAcMAk5MMREwDwYDVQQKDAhQb2xhclNTTDEZMBcG +A1UEAwwQUG9sYXJTU0wgVGVzdCBDQTAeFw0yMTExMDkwNTA4MDhaFw0zMTExMDcw +NTA4MDhaMFUxCzAJBgNVBAYTAk5MMQswCQYDVQQIDAJOTDELMAkGA1UEBwwCTkwx +ETAPBgNVBAoMCFBvbGFyU1NMMRkwFwYDVQQDDBBQb2xhclNTTCBUZXN0IENBMIIC +IjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAt4LPFNvOHrkgztHoZ9LR+69Q +uhDzaPmlEGngmJSgxMZi5dJJ7ZnEJtZ+8bjX/DTCZcl7lOwwSHurb9a3GPgP/DRt +0jGqqtDiwZnc1OoJ9mLnj+BoB1hhWSMuIcJ+cM292Aac+12Ih9mThtzhbxzW/6xD +neyAtRjF/iRBNq3FIiAMj3c+w+BBVmEbasBpuXs/4S9BTvyPrGFjg+jKf842wTwG +Vz82goPKaOMByRWbhEKhYT9al/iKcdPIPjzA6HB3XyPjLm9xpIXR1+RidO98UhsC +Llzw8xeUi+aq2MHLilBEUsHqVoJqWqFOCfJhOQGFzPQNPnWk1l7FOvJvYSr4XR1/ +6eIUqzFlG30ImG7uVCzhLunk092aPNGvewogTLiprmeLLBAks70NzPD9W1UQhXhB +SqZN6FNpczAdn5x+UxCUQkmFs8siCtLbh7Z3H2lx34IaiH9Xw+YTV244J/5UuUBR +jWrncbM+XElXo4RzF3gIL4EmvxKTzEWWh3b6SNpvirdJFbSkwByOYfBZ3XUuQDsa +Df9UGAgD1EBHwpuZB9VYT1rVA0x9o2ovjwYXY4AjHRCgqe53FeUSyxyY/xXKqiPn +jJjZuo2Pif2W+Il14Me1e30SD8+8fbdfoaPiGYQ02aW5AvElbMk4KEGnIlgiYhs9 +qAn36FqlaOkiwI7CQH8CAwEAAaNTMFEwHQYDVR0OBBYEFIo1oGRr0T8oklUSbVpt +/e7wKtXbMB8GA1UdIwQYMBaAFIo1oGRr0T8oklUSbVpt/e7wKtXbMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAHkCHYagsPWfOipq6vXqaaCmHaaY +dOao9BjawfZbZOrYXFYFDcSVP4iGltEs3NBBrzjkbqmRlDT4R3uvWuvD+TlIUroL +0eGjPwmljGqW2eokjuySf9V48PdH12xwFnSYRn/qFiiEB7LconoxGhtXLOF1ry0q +99d4Lk/8pUlOVWFCQ5kv+Fp27oungmS6n4H2973HbzRbm8rxQJrrtomvQcU5DgN3 +b6q+37VVprE5dGJZ/+XCZ85lbOMnvN3H8zd1rftFnU2fxNK8DZ8cqb//BOI5lWpg +wBc8zUNCrNnPb8An/xpbS1P0BUGxxSWeyiXOwv9mxc3x7zJLVrDQLVd6GbhBnLE6 +Z9aC2Eh8gNLvBhs/JVaQKPn3iVyqNAaXaOg90TnlliO9EVbA0212HyC2+ShbdkwE +9TEDP9QPAr8Y/tU0hCpALelwE8VZJv1rp71Ga6CivlfDcgBoVygAQQ8tH+PcS+u6 +1r+B0TPml86mUAzG5lAW+ZuSjw727aLlLOnk/7F8jOShz817lA6eMqtlzYZpM+lR +B8ZNdkuuQq3z8OkJ78Gjby4NeDguF/dMlXD5me+AJ4FMqeCEN3g6QKh8ThzoSp8h +ZxmK9bX/MagiL0upybyqTDqgNCKNkpcqY168Nborx/T5J4srsBdJoCDi3jO1Q3nk +UvFuODojjTxO9Bwa +-----END CERTIFICATE----- From 26665c7b4830cd0bac336847b30f5a26d28ebdc8 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Wed, 17 Nov 2021 14:09:54 +0530 Subject: [PATCH 10/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- examples/aks-attestation/README.md | 22 ++++++++++--------- .../base-image-generation-script.sh | 5 ++++- examples/aks-attestation/certs/README | 4 ++++ 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/examples/aks-attestation/README.md b/examples/aks-attestation/README.md index 3070c948..2f1a241a 100644 --- a/examples/aks-attestation/README.md +++ b/examples/aks-attestation/README.md @@ -10,16 +10,18 @@ deployed to the AKS cluster. ## Preparing client and server images -This demonstration is created for ``gramine/CI-Examples/ra-tls-secret-prov`` sample. The sample -contains client and server applications, where by-default server is running on localhost:4433. Here, -the client sends its SGX quote to the server for verification. After successful quote verification, -the server sends a secret to the client. To run these client and server applications inside AKS -cluster, user needs to prepare two docker images, each for client and server application. Since, now -the server will no longer run on localhost, instead it will run as part of a container inside AKS -cluster, the server container should be assigned a dns name (e.g., ``) for outside -container visibility. The client will send requests to this dns name. Therefore, for demonstration - we updated ``gramine/CI-Examples/ra-tls-secret-prov/certs`` directory certificates by replacing -"Common Name" field in the server certificate (i.e., `server2-sha256.crt`) from `localhost` to +This demonstration is based on ra-tls-secret-prov sample from +``gramine/CI-Examples/ra-tls-secret-prov``. Familiarity with this sample is highly recommended +before proceeding further. The sample contains client and server applications, where by-default +server is running on localhost:4433. Here, the client sends its SGX quote to the server for +verification. After successful quote verification, the server sends a secret to the client. To run +these client and server applications inside AKS cluster, user needs to prepare two docker images, +each for client and server application. Since, now the server will no longer run on localhost, +instead it will run as part of a container inside AKS cluster, the server container should be +assigned a dns name (e.g., ``) for outside container visibility. The client will send +requests to this dns name. Therefore, for demonstration we updated +``gramine/CI-Examples/ra-tls-secret-prov/certs`` directory certificates by replacing "Common Name" +field in the server certificate (i.e., `server2-sha256.crt`) from `localhost` to ``. In order to create base client and server images for AKS environment, user can execute diff --git a/examples/aks-attestation/base-image-generation-script.sh b/examples/aks-attestation/base-image-generation-script.sh index db94cd9b..f5711474 100755 --- a/examples/aks-attestation/base-image-generation-script.sh +++ b/examples/aks-attestation/base-image-generation-script.sh @@ -1,3 +1,6 @@ +# This script assumes that in-kernel driver is installed on the host system. +# Please refer https://gramine.readthedocs.io/en/latest/building.html#id2 for more details. + # install Gramine dependencies apt-get install -y \ @@ -30,7 +33,7 @@ git clone https://github.com/gramineproject/gramine.git cd gramine/Pal/src/host/Linux-SGX/signer/ openssl genrsa -3 -out enclave-key.pem 3072 -# Build Gramine with DCAP enabled mode +# Build Gramine with DCAP enabled mode (assuming in-kernel driver) cd ../../../../../ meson setup build/ --buildtype=release -Ddirect=enabled -Dsgx=enabled -Ddcap=enabled diff --git a/examples/aks-attestation/certs/README b/examples/aks-attestation/certs/README index 8877aa05..84041afe 100644 --- a/examples/aks-attestation/certs/README +++ b/examples/aks-attestation/certs/README @@ -6,3 +6,7 @@ This directory contains pre-generated example certificates. In particular: Loaded in server (verifier), so it will send it to the client during TLS handshake. Common Name `ra-tls-server-aks-dns.eastus.cloudapp.azure.com` - `server2.key` -- RSA private key in PEM format. Loaded in server (verifier). + +Note: These certificates should not be used in production. Here, user should + update the server2-sha256.crt with ``Common Name`` pointing to the dns + name of the ra-tls-secret-prov server deployed at AKS cluster. From f8ecf6f3acd0ee5f4bc9dc672ce0955b6ecefcc3 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Wed, 17 Nov 2021 15:00:29 +0530 Subject: [PATCH 11/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- examples/aks-attestation/base-image-generation-script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/aks-attestation/base-image-generation-script.sh b/examples/aks-attestation/base-image-generation-script.sh index f5711474..2e650a39 100755 --- a/examples/aks-attestation/base-image-generation-script.sh +++ b/examples/aks-attestation/base-image-generation-script.sh @@ -61,4 +61,4 @@ docker build -f aks-secret-prov-client.dockerfile -t aks-secret-prov-client-img # Remove Gramine directory -rm -r gramine/ +rm -rf gramine/ From 019424984c6dd46ef592c42c52175d379b88688c Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Thu, 18 Nov 2021 17:02:16 +0530 Subject: [PATCH 12/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- examples/aks-attestation/README.md | 4 +- .../base-image-generation-script.sh | 55 +++---------------- examples/aks-attestation/gramine_build.sh | 46 ++++++++++++++++ 3 files changed, 57 insertions(+), 48 deletions(-) create mode 100755 examples/aks-attestation/gramine_build.sh diff --git a/examples/aks-attestation/README.md b/examples/aks-attestation/README.md index 2f1a241a..5910eae0 100644 --- a/examples/aks-attestation/README.md +++ b/examples/aks-attestation/README.md @@ -25,8 +25,8 @@ field in the server certificate (i.e., `server2-sha256.crt`) from `localhost` to ``. In order to create base client and server images for AKS environment, user can execute -base-image-generation-script.sh script. Since, both client and server applications will run -inside containers in AKS cluster, and the client wants to send its SGX quote to the server for +base-image-generation-script.sh script (with sudo). Since, both client and server applications will +run inside containers in AKS cluster, and the client wants to send its SGX quote to the server for verification, therefore the user needs to graminize the client application, so that it can leverage SGX capabilities from within a container. Hence, the following two steps create base server image and gsc-client image for AKS cluster. diff --git a/examples/aks-attestation/base-image-generation-script.sh b/examples/aks-attestation/base-image-generation-script.sh index 2e650a39..168bd6c3 100755 --- a/examples/aks-attestation/base-image-generation-script.sh +++ b/examples/aks-attestation/base-image-generation-script.sh @@ -1,53 +1,16 @@ -# This script assumes that in-kernel driver is installed on the host system. -# Please refer https://gramine.readthedocs.io/en/latest/building.html#id2 for more details. +# Download and build Gramine directory -# install Gramine dependencies - -apt-get install -y \ - autoconf \ - bison \ - build-essential \ - coreutils \ - curl \ - gawk \ - git \ - libcurl4-openssl-dev \ - libprotobuf-c-dev \ - linux-headers-generic \ - ninja-build \ - pkg-config \ - protobuf-c-compiler \ - python3 \ - python3-pip \ - python3-protobuf \ - wget - -python3 -B -m pip install 'toml>=0.10' 'meson>=0.55' - -# Download Gramine - -git clone https://github.com/gramineproject/gramine.git - -# Generate Signing Key - -cd gramine/Pal/src/host/Linux-SGX/signer/ -openssl genrsa -3 -out enclave-key.pem 3072 - -# Build Gramine with DCAP enabled mode (assuming in-kernel driver) - -cd ../../../../../ -meson setup build/ --buildtype=release -Ddirect=enabled -Dsgx=enabled -Ddcap=enabled -ninja -C build/ -sudo ninja -C build/ install - -# Copy dummy server certificate with Common Name as " - -cd CI-Examples/ra-tls-secret-prov -mv certs certs_orig -cp -r ../../../certs ./ +gramine_dir="gramine" +if [ -d "$gramine_dir" ]; then + echo "\n\n ***** '$gramine_dir' directory exists, proceeding to image generation ***** \n\n" +else + chmod u+x gramine_build.sh + ./gramine_build.sh +fi # Create Server image +cd gramine/CI-Examples/ra-tls-secret-prov make clean && make dcap cd ../../../ docker build -f aks-secret-prov-server.dockerfile -t aks-secret-prov-server-img . diff --git a/examples/aks-attestation/gramine_build.sh b/examples/aks-attestation/gramine_build.sh new file mode 100755 index 00000000..7f7248c7 --- /dev/null +++ b/examples/aks-attestation/gramine_build.sh @@ -0,0 +1,46 @@ +# This script assumes that in-kernel driver is installed on the host system. +# Please refer https://gramine.readthedocs.io/en/latest/building.html#id2 for more details. + +# install Gramine dependencies + +apt-get install -y \ + autoconf \ + bison \ + build-essential \ + coreutils \ + curl \ + gawk \ + git \ + libcurl4-openssl-dev \ + libprotobuf-c-dev \ + linux-headers-generic \ + ninja-build \ + pkg-config \ + protobuf-c-compiler \ + python3 \ + python3-pip \ + python3-protobuf \ + wget + +python3 -B -m pip install 'toml>=0.10' 'meson>=0.55' + +# Download Gramine + +git clone https://github.com/gramineproject/gramine.git + +# Generate Signing Key + +cd gramine/Pal/src/host/Linux-SGX/signer/ +openssl genrsa -3 -out enclave-key.pem 3072 + +# Build Gramine with DCAP enabled mode (assuming in-kernel driver) + +cd ../../../../../ +meson setup build/ --buildtype=release -Ddirect=enabled -Dsgx=enabled -Ddcap=enabled +ninja -C build/ +sudo ninja -C build/ install + +# Copy dummy server certificate with Common Name as " +cd CI-Examples/ra-tls-secret-prov +mv certs certs_orig +cp -r ../../../certs ./ From 0596f835fcaaca5dabb61b6a9dd9be5895176bb9 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Thu, 18 Nov 2021 18:24:04 +0530 Subject: [PATCH 13/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- examples/aks-attestation/README.md | 9 +++++---- .../aks-attestation/aks-secret-prov-server.dockerfile | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/aks-attestation/README.md b/examples/aks-attestation/README.md index 5910eae0..3ec9dcff 100644 --- a/examples/aks-attestation/README.md +++ b/examples/aks-attestation/README.md @@ -80,13 +80,14 @@ and gsc-client image for AKS cluster. ## Deploying client and server images inside AKS Confidential Compute cluster AKS confidential compute cluster can be created using following -[link](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-nodes-aks-get-started). +[link](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-enclave-nodes-aks-get-started). Gramine performs out-of-proc mode DCAP quote generation. Out-of-proc mode quote generation requires aesmd service. To fulfill this requirement, AKS provides -[sgxquotehelper daemonset](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-nodes-out-of-proc-attestation). -This feature exposes aesmd service for the container node. The service will internally connect with -az-dcap-client to fetch the platform collateral required for quote generation. In this demo, the +[sgxquotehelper daemonset](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-nodes-aks-addon#out-of-proc-attestation-for-confidential-workloads) +(can be enabled by ``--enable-sgxquotehelper`` during cluster creation). This feature exposes aesmd +service for the container node. The service will internally connect with az-dcap-client to fetch the +platform collateral required for quote generation. In this demo, the ``aks-secret-prov-client-deployment.yaml`` uses aesmd service exposed by AKS with the help of sgxquotehelper plugin. diff --git a/examples/aks-attestation/aks-secret-prov-server.dockerfile b/examples/aks-attestation/aks-secret-prov-server.dockerfile index 1d2318ef..e857cbb5 100644 --- a/examples/aks-attestation/aks-secret-prov-server.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-server.dockerfile @@ -14,8 +14,8 @@ RUN apt-get update \ # version used for quote generation. User can replace the below package with the # latest package. -RUN wget https://github.com/microsoft/Azure-DCAP-Client/releases/download/1.8/az-dcap-client_1.8_amd64_18.04.deb \ - && dpkg -i az-dcap-client_1.8_amd64_18.04.deb +RUN wget https://packages.microsoft.com/ubuntu/18.04/prod/pool/main/a/az-dcap-client/az-dcap-client_1.10_amd64.deb \ + && dpkg -i az-dcap-client_1.10_amd64.deb # Installing DCAP Quote Verification Library RUN echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' \ From 11a24b6b7a3a7bb05ad752f7d3d05dccb31701f4 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Thu, 18 Nov 2021 19:31:01 +0530 Subject: [PATCH 14/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- examples/aks-attestation/README.md | 1 + .../aks-attestation/aks-secret-prov-server-deployment.yaml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/examples/aks-attestation/README.md b/examples/aks-attestation/README.md index 3ec9dcff..64c8f260 100644 --- a/examples/aks-attestation/README.md +++ b/examples/aks-attestation/README.md @@ -57,6 +57,7 @@ and gsc-client image for AKS cluster. ```sh $ cd gsc + $ cp config.yaml.template config.yaml (we tested aks-gsc-secret-prov-client-img for DCAP 1.11) $ openssl genrsa -3 -out enclave-key.pem 3072 $ ./gsc build \ examples/aks-attestation/aks-secret-prov-client.manifest diff --git a/examples/aks-attestation/aks-secret-prov-server-deployment.yaml b/examples/aks-attestation/aks-secret-prov-server-deployment.yaml index 8560a9ff..29857c7b 100644 --- a/examples/aks-attestation/aks-secret-prov-server-deployment.yaml +++ b/examples/aks-attestation/aks-secret-prov-server-deployment.yaml @@ -20,6 +20,10 @@ spec: resources: limits: kubernetes.azure.com/sgx_epc_mem_in_MiB: 10 + +# TODO: Requesting an SGX machine is not needed, but Intel DCAP libraries have a bug of trying to +# open the SGX driver (see https://github.com/intel/linux-sgx/issues/756) + --- apiVersion: v1 kind: Service From b264d1367cd088b0650d2b5cf72d1b1f3654e4f1 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Thu, 25 Nov 2021 12:33:06 +0530 Subject: [PATCH 15/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- .../aks-secret-prov-client.dockerfile | 6 ++---- .../aks-secret-prov-server.dockerfile | 20 ++++++++----------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/examples/aks-attestation/aks-secret-prov-client.dockerfile b/examples/aks-attestation/aks-secret-prov-client.dockerfile index 27d7df30..f5f012d9 100644 --- a/examples/aks-attestation/aks-secret-prov-client.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-client.dockerfile @@ -19,12 +19,10 @@ RUN apt-get update \ libsgx-dcap-ql \ libsgx-quote-ex -RUN mkdir -p /ra-tls-secret-prov +WORKDIR /ra-tls-secret-prov -COPY gramine/CI-Examples/ra-tls-secret-prov /ra-tls-secret-prov +COPY gramine/CI-Examples/ra-tls-secret-prov . COPY gramine/CI-Examples/ra-tls-secret-prov/secret_prov_min_client /usr/local/bin -WORKDIR /ra-tls-secret-prov - ENTRYPOINT ["secret_prov_min_client"] diff --git a/examples/aks-attestation/aks-secret-prov-server.dockerfile b/examples/aks-attestation/aks-secret-prov-server.dockerfile index e857cbb5..9a2c5f4c 100644 --- a/examples/aks-attestation/aks-secret-prov-server.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-server.dockerfile @@ -25,23 +25,19 @@ RUN echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bio RUN apt-get update && apt-get install -y libsgx-dcap-quote-verify -# Build environment of this Dockerfile should point to the root of Gramine directory - -RUN mkdir -p /ra-tls-secret-prov +WORKDIR /ra-tls-secret-prov -COPY gramine/CI-Examples/ra-tls-secret-prov /ra-tls-secret-prov +COPY gramine/CI-Examples/ra-tls-secret-prov . COPY gramine/CI-Examples/ra-tls-secret-prov/secret_prov_server_dcap /usr/local/bin -RUN mkdir -p /ra-tls-secret-prov/libs +RUN mkdir libs -COPY gramine/build/Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_verify_dcap.so /ra-tls-secret-prov/libs -COPY gramine/build/Pal/src/host/Linux-SGX/tools/common/libsgx_util.so /ra-tls-secret-prov/libs -COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedcrypto_gramine.so.6 /ra-tls-secret-prov/libs -COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedtls_gramine.so.13 /ra-tls-secret-prov/libs -COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedx509_gramine.so.1 /ra-tls-secret-prov/libs - -WORKDIR /ra-tls-secret-prov +COPY gramine/build/Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_verify_dcap.so libs +COPY gramine/build/Pal/src/host/Linux-SGX/tools/common/libsgx_util.so libs +COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedcrypto_gramine.so.6 libs +COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedtls_gramine.so.13 libs +COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedx509_gramine.so.1 libs ENV LD_LIBRARY_PATH = "${LD_LIBRARY_PATH}:/ra-tls-secret-prov/libs" From 3dd139d773c09eaf3e1aa8f2bdf9d26d09d20db7 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Mon, 29 Nov 2021 13:55:26 +0530 Subject: [PATCH 16/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- examples/aks-attestation/README.md | 99 +++++++++---------- .../aks-secret-prov-client-deployment.yaml | 2 +- .../aks-secret-prov-client.dockerfile | 11 ++- .../aks-secret-prov-client.manifest | 3 - .../aks-secret-prov-server-deployment.yaml | 2 +- .../aks-secret-prov-server.dockerfile | 3 +- .../base-image-generation-script.sh | 7 +- examples/aks-attestation/certs/README | 5 +- examples/aks-attestation/gramine_build.sh | 16 ++- 9 files changed, 75 insertions(+), 73 deletions(-) diff --git a/examples/aks-attestation/README.md b/examples/aks-attestation/README.md index 64c8f260..90364dfe 100644 --- a/examples/aks-attestation/README.md +++ b/examples/aks-attestation/README.md @@ -3,82 +3,83 @@ This guide demonstrates how Gramine DCAP attestation quote can be generated and verified from within an AKS cluster. Here, we provide an end-to-end example to help Cloud Solution Providers integrate Gramine’s RA-TLS attestation and secret provisioning feature with a confidential compute -cluster managed by Azure Kubernetes Service. The necessary reference wrappers that will enable -Gramine to use AKS components such as the AESMD and quote provider libraries are contributed. -A microservice deployment is also provided for the RA-TLS verifier module that can be readily +cluster managed by Azure Kubernetes Service. This guide contains necessary reference wrappers that +enable Gramine to use AKS components such as AESMD and DCAP quote provider libraries. This guide +also describes a microservice deployment for the RA-TLS verifier (server) that can be readily deployed to the AKS cluster. ## Preparing client and server images -This demonstration is based on ra-tls-secret-prov sample from -``gramine/CI-Examples/ra-tls-secret-prov``. Familiarity with this sample is highly recommended -before proceeding further. The sample contains client and server applications, where by-default -server is running on localhost:4433. Here, the client sends its SGX quote to the server for -verification. After successful quote verification, the server sends a secret to the client. To run -these client and server applications inside AKS cluster, user needs to prepare two docker images, -each for client and server application. Since, now the server will no longer run on localhost, -instead it will run as part of a container inside AKS cluster, the server container should be -assigned a dns name (e.g., ``) for outside container visibility. The client will send -requests to this dns name. Therefore, for demonstration we updated -``gramine/CI-Examples/ra-tls-secret-prov/certs`` directory certificates by replacing "Common Name" -field in the server certificate (i.e., `server2-sha256.crt`) from `localhost` to -``. - -In order to create base client and server images for AKS environment, user can execute -base-image-generation-script.sh script (with sudo). Since, both client and server applications will -run inside containers in AKS cluster, and the client wants to send its SGX quote to the server for -verification, therefore the user needs to graminize the client application, so that it can leverage -SGX capabilities from within a container. Hence, the following two steps create base server image -and gsc-client image for AKS cluster. +This demonstration is based on the ra-tls-secret-prov example from +https://github.com/gramineproject/gramine/tree/master/CI-Examples/ra-tls-secret-prov. Familiarity +with this example is highly recommended before proceeding further. The sample contains client and +server applications, where by default server is running on localhost:4433. In the example, the +client sends its SGX quote to the server for verification. After successful quote verification, the +server sends a secret to the client. To run these client and server applications inside the AKS +cluster, user needs to prepare two docker images, one for the client and one for the server. In our +AKS attestation example, the server will no longer run on localhost, instead it will run in a Docker +container inside the AKS cluster. The server container should be assigned a DNS name +(e.g., ``) to be accessible from the outside of the container. The client will send +requests to this DNS name. Therefore, for demonstration we updated the example certificates from +https://github.com/gramineproject/gramine/tree/master/CI-Examples/ra-tls-secret-prov/certs by +replacing "Common Name" field in the server certificate (i.e., `server2-sha256.crt`) from +`localhost` to ``. + +In order to create base client and server images for the AKS environment, user can execute +`base-image-generation-script.sh` script (with sudo). Since both client and server applications will +run inside containers in the AKS cluster, and the client application will send its SGX quote to the +server for verification, therefore the user needs to graminize the client application. Hence, the +following two steps create a native Docker server image and a graminized GSC client image for the +AKS cluster. ### Creating server image -1. The base-image-generation-script.sh script will create server image with the name - aks-secret-prov-server-img:latest. +1. The `base-image-generation-script.sh` script will create the native Docker server image with the + name `aks-secret-prov-server-img:latest`. 2. Push server image to Docker Hub or your preferred registry: ```sh - $ docker tag \ - / - $ docker push / + $ docker tag aks-secret-prov-server-img:latest \ + /aks-secret-prov-server-img:latest + $ docker push /aks-secret-prov-server-img:latest ``` -3. Deploy `` in AKS confidential compute cluster: +3. Deploy `aks-secret-prov-server-img:latest` in AKS confidential compute cluster: - Reference deployment file: `gsc/examples/aks-attestation/aks-secret-prov-server-deployment.yaml` ### Creating client image -1. The base-image-generation-script.sh script will create client image with the name - aks-secret-prov-client-img:latest. +1. The `base-image-generation-script.sh` script will create the native Docker client image with the + name `aks-secret-prov-client-img:latest`. -2. Create GSC image for ra-tls-secret-prov min client: +2. Create the GSC client image: ```sh $ cd gsc - $ cp config.yaml.template config.yaml (we tested aks-gsc-secret-prov-client-img for DCAP 1.11) + $ cp config.yaml.template config.yaml $ openssl genrsa -3 -out enclave-key.pem 3072 - $ ./gsc build \ + $ ./gsc build aks-secret-prov-client-img:latest \ examples/aks-attestation/aks-secret-prov-client.manifest - $ ./gsc sign-image enclave-key.pem + $ ./gsc sign-image aks-secret-prov-client-img:latest enclave-key.pem ``` 5. Push resulting image to Docker Hub or your preferred registry: ```sh - $ docker tag \ - / - $ docker push / + $ docker tag gsc-aks-secret-prov-client-img:latest \ + /gsc-aks-secret-prov-client-img:latest + $ docker push /gsc-aks-secret-prov-client-img:latest ``` -6. Deploy `` in AKS confidential compute cluster: +6. Deploy `gsc-aks-secret-prov-client-img:latest` in AKS confidential compute cluster: - Reference deployment file: `gsc/examples/aks-attestation/aks-secret-prov-client-deployment.yaml` -**NOTE**: We recommend deploying GSC images on Ubuntu with Linux kernel version 5.11 or higher. +**NOTE**: We tested this example with DCAP driver 1.11 specified in the GSC configuration file. -## Deploying client and server images inside AKS Confidential Compute cluster +## Deploying client and server images in AKS Confidential Compute cluster AKS confidential compute cluster can be created using following [link](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-enclave-nodes-aks-get-started). @@ -86,17 +87,15 @@ AKS confidential compute cluster can be created using following Gramine performs out-of-proc mode DCAP quote generation. Out-of-proc mode quote generation requires aesmd service. To fulfill this requirement, AKS provides [sgxquotehelper daemonset](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-nodes-aks-addon#out-of-proc-attestation-for-confidential-workloads) -(can be enabled by ``--enable-sgxquotehelper`` during cluster creation). This feature exposes aesmd +(can be enabled by `--enable-sgxquotehelper` during cluster creation). This feature exposes aesmd service for the container node. The service will internally connect with az-dcap-client to fetch the platform collateral required for quote generation. In this demo, the -``aks-secret-prov-client-deployment.yaml`` uses aesmd service exposed by AKS with the help of +`aks-secret-prov-client-deployment.yaml` uses aesmd service exposed by AKS with the help of sgxquotehelper plugin. In the ra-tls-secret-prov example, the client will generate out-of-proc mode sgx quote that will be embedded inside RA-TLS certificate. On receiving the quote, the server will internally verify it -using libsgx-dcap-quote-verify library via az-dcap-client library. Here, -``aks-secret-prov-server-deployment.yaml`` will deploy a ra-tls-secret-prov server container inside - AKS cluster. +using libsgx-dcap-quote-verify library via az-dcap-client library. ### Deployment @@ -110,8 +109,8 @@ Once the server container is in running state, start the client container as sho $ kubectl apply -f aks-secret-prov-client-deployment.yaml ``` -At this stage, a successful RA-TLS verification would be completed, and the secrets have been -provisioned from the server to the client container. +At this stage, a successful RA-TLS verification will be completed, and the secrets will be +provisioned from the server to the client. ## Checking SGX quote generation and verification @@ -127,11 +126,11 @@ Receive logs to verify the secret has been provisioned to the client: $ kubectl logs -l app=gsc-ra-tls-secret-prov-client --tail=50 ``` -### Expected Output +Expected output: `--- Received secret = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'` -Delete both client and server containers +Delete both client and server containers: ```sh $ kubectl delete -f aks-secret-prov-server-deployment.yaml diff --git a/examples/aks-attestation/aks-secret-prov-client-deployment.yaml b/examples/aks-attestation/aks-secret-prov-client-deployment.yaml index bba5ecbf..ac875be2 100644 --- a/examples/aks-attestation/aks-secret-prov-client-deployment.yaml +++ b/examples/aks-attestation/aks-secret-prov-client-deployment.yaml @@ -16,7 +16,7 @@ spec: path: /var/run/aesmd containers: - name: gsc-ra-tls-secret-prov-client-container - image: / + image: /gsc-aks-secret-prov-client-img:latest env: - name: SECRET_PROVISION_SERVERS value: ":4433" diff --git a/examples/aks-attestation/aks-secret-prov-client.dockerfile b/examples/aks-attestation/aks-secret-prov-client.dockerfile index f5f012d9..c8f45683 100644 --- a/examples/aks-attestation/aks-secret-prov-client.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-client.dockerfile @@ -21,8 +21,17 @@ RUN apt-get update \ WORKDIR /ra-tls-secret-prov -COPY gramine/CI-Examples/ra-tls-secret-prov . +COPY gramine/CI-Examples/ra-tls-secret-prov/certs ./certs COPY gramine/CI-Examples/ra-tls-secret-prov/secret_prov_min_client /usr/local/bin +RUN mkdir libs + +COPY gramine/build/Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_attest.so libs +COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedcrypto_gramine.so.6 libs +COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedtls_gramine.so.13 libs +COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedx509_gramine.so.1 libs + +ENV LD_LIBRARY_PATH = "${LD_LIBRARY_PATH}:/ra-tls-secret-prov/libs" + ENTRYPOINT ["secret_prov_min_client"] diff --git a/examples/aks-attestation/aks-secret-prov-client.manifest b/examples/aks-attestation/aks-secret-prov-client.manifest index d7b77bf0..22d5f6d7 100644 --- a/examples/aks-attestation/aks-secret-prov-client.manifest +++ b/examples/aks-attestation/aks-secret-prov-client.manifest @@ -1,5 +1,3 @@ -# Manifest file for ra-tls-secret-prov min client - # Secret Provisioning library (client-side) is preloaded loader.env.LD_PRELOAD = "libsecret_prov_attest.so" @@ -7,7 +5,6 @@ loader.env.SECRET_PROVISION_SERVERS = { passthrough = true } loader.env.SECRET_PROVISION_CONSTRUCTOR = "1" loader.env.SECRET_PROVISION_CA_CHAIN_PATH = "certs/test-ca-sha256.crt" -# Request remote attestation functionality from Gramine sgx.remote_attestation = true sgx.allowed_files = [ diff --git a/examples/aks-attestation/aks-secret-prov-server-deployment.yaml b/examples/aks-attestation/aks-secret-prov-server-deployment.yaml index 29857c7b..9a43cbed 100644 --- a/examples/aks-attestation/aks-secret-prov-server-deployment.yaml +++ b/examples/aks-attestation/aks-secret-prov-server-deployment.yaml @@ -14,7 +14,7 @@ spec: spec: containers: - name: ra-tls-secret-prov-server-container - image: / + image: /aks-secret-prov-server-img:latest ports: - containerPort: 4433 resources: diff --git a/examples/aks-attestation/aks-secret-prov-server.dockerfile b/examples/aks-attestation/aks-secret-prov-server.dockerfile index 9a2c5f4c..12b57130 100644 --- a/examples/aks-attestation/aks-secret-prov-server.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-server.dockerfile @@ -27,7 +27,8 @@ RUN apt-get update && apt-get install -y libsgx-dcap-quote-verify WORKDIR /ra-tls-secret-prov -COPY gramine/CI-Examples/ra-tls-secret-prov . +COPY gramine/CI-Examples/ra-tls-secret-prov/certs ./certs +COPY gramine/CI-Examples/ra-tls-secret-prov/files ./files COPY gramine/CI-Examples/ra-tls-secret-prov/secret_prov_server_dcap /usr/local/bin diff --git a/examples/aks-attestation/base-image-generation-script.sh b/examples/aks-attestation/base-image-generation-script.sh index 168bd6c3..52fbed6d 100755 --- a/examples/aks-attestation/base-image-generation-script.sh +++ b/examples/aks-attestation/base-image-generation-script.sh @@ -1,8 +1,7 @@ # Download and build Gramine directory -gramine_dir="gramine" -if [ -d "$gramine_dir" ]; then - echo "\n\n ***** '$gramine_dir' directory exists, proceeding to image generation ***** \n\n" +if [ -d "gramine" ]; then + echo "\n\n ***** gramine directory exists, proceeding to image generation ***** \n\n" else chmod u+x gramine_build.sh ./gramine_build.sh @@ -22,6 +21,4 @@ make clean && make secret_prov_min_client cd ../../../ docker build -f aks-secret-prov-client.dockerfile -t aks-secret-prov-client-img . -# Remove Gramine directory - rm -rf gramine/ diff --git a/examples/aks-attestation/certs/README b/examples/aks-attestation/certs/README index 84041afe..1c1ae35f 100644 --- a/examples/aks-attestation/certs/README +++ b/examples/aks-attestation/certs/README @@ -4,9 +4,10 @@ This directory contains pre-generated example certificates. In particular: in client (attester) so that client can verify the server's certificate. - `server2-sha256.crt` -- RSA SHA256 leaf server certificate in PEM format. Loaded in server (verifier), so it will send it to the client during TLS - handshake. Common Name `ra-tls-server-aks-dns.eastus.cloudapp.azure.com` + handshake. The Common Name field is set to + `ra-tls-server-aks-dns.eastus.cloudapp.azure.com`. - `server2.key` -- RSA private key in PEM format. Loaded in server (verifier). Note: These certificates should not be used in production. Here, user should - update the server2-sha256.crt with ``Common Name`` pointing to the dns + update the server2-sha256.crt with ``Common Name`` pointing to the DNS name of the ra-tls-secret-prov server deployed at AKS cluster. diff --git a/examples/aks-attestation/gramine_build.sh b/examples/aks-attestation/gramine_build.sh index 7f7248c7..6f0047a0 100755 --- a/examples/aks-attestation/gramine_build.sh +++ b/examples/aks-attestation/gramine_build.sh @@ -1,9 +1,9 @@ # This script assumes that in-kernel driver is installed on the host system. -# Please refer https://gramine.readthedocs.io/en/latest/building.html#id2 for more details. +# Please refer to https://gramine.readthedocs.io/en/latest/building.html#id2 for more details. # install Gramine dependencies -apt-get install -y \ +sudo apt-get install -y \ autoconf \ bison \ build-essential \ @@ -22,7 +22,7 @@ apt-get install -y \ python3-protobuf \ wget -python3 -B -m pip install 'toml>=0.10' 'meson>=0.55' +sudo python3 -B -m pip install 'toml>=0.10' 'meson>=0.55' # Download Gramine @@ -30,17 +30,15 @@ git clone https://github.com/gramineproject/gramine.git # Generate Signing Key -cd gramine/Pal/src/host/Linux-SGX/signer/ -openssl genrsa -3 -out enclave-key.pem 3072 +cd gramine +openssl genrsa -3 -out Pal/src/host/Linux-SGX/signer/enclave-key.pem 3072 # Build Gramine with DCAP enabled mode (assuming in-kernel driver) -cd ../../../../../ meson setup build/ --buildtype=release -Ddirect=enabled -Dsgx=enabled -Ddcap=enabled ninja -C build/ sudo ninja -C build/ install # Copy dummy server certificate with Common Name as " -cd CI-Examples/ra-tls-secret-prov -mv certs certs_orig -cp -r ../../../certs ./ +cd ../ +cp -r certs/ gramine/CI-Examples/ra-tls-secret-prov/ From 2c4905bbb76028fdec73d14737a94934cf977b38 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Mon, 29 Nov 2021 15:09:15 +0530 Subject: [PATCH 17/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- .../aks-attestation/aks-secret-prov-client.dockerfile | 9 --------- 1 file changed, 9 deletions(-) diff --git a/examples/aks-attestation/aks-secret-prov-client.dockerfile b/examples/aks-attestation/aks-secret-prov-client.dockerfile index c8f45683..3275d8a0 100644 --- a/examples/aks-attestation/aks-secret-prov-client.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-client.dockerfile @@ -25,13 +25,4 @@ COPY gramine/CI-Examples/ra-tls-secret-prov/certs ./certs COPY gramine/CI-Examples/ra-tls-secret-prov/secret_prov_min_client /usr/local/bin -RUN mkdir libs - -COPY gramine/build/Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_attest.so libs -COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedcrypto_gramine.so.6 libs -COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedtls_gramine.so.13 libs -COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedx509_gramine.so.1 libs - -ENV LD_LIBRARY_PATH = "${LD_LIBRARY_PATH}:/ra-tls-secret-prov/libs" - ENTRYPOINT ["secret_prov_min_client"] From 07fd80d3f26279680c0e948c29ad0deb2619090f Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Tue, 30 Nov 2021 13:07:13 +0530 Subject: [PATCH 18/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- examples/aks-attestation/.gitignore | 1 + examples/aks-attestation/README.md | 28 +++++++++---------- .../base-image-generation-script.sh | 3 +- examples/aks-attestation/certs/README | 4 +-- examples/aks-attestation/gramine_build.sh | 2 +- 5 files changed, 19 insertions(+), 19 deletions(-) create mode 100644 examples/aks-attestation/.gitignore diff --git a/examples/aks-attestation/.gitignore b/examples/aks-attestation/.gitignore new file mode 100644 index 00000000..ad83f82c --- /dev/null +++ b/examples/aks-attestation/.gitignore @@ -0,0 +1 @@ +/gramine diff --git a/examples/aks-attestation/README.md b/examples/aks-attestation/README.md index 90364dfe..cc6814ce 100644 --- a/examples/aks-attestation/README.md +++ b/examples/aks-attestation/README.md @@ -10,7 +10,7 @@ deployed to the AKS cluster. ## Preparing client and server images -This demonstration is based on the ra-tls-secret-prov example from +This demonstration is based on the `ra-tls-secret-prov` example from https://github.com/gramineproject/gramine/tree/master/CI-Examples/ra-tls-secret-prov. Familiarity with this example is highly recommended before proceeding further. The sample contains client and server applications, where by default server is running on localhost:4433. In the example, the @@ -22,10 +22,10 @@ container inside the AKS cluster. The server container should be assigned a DNS (e.g., ``) to be accessible from the outside of the container. The client will send requests to this DNS name. Therefore, for demonstration we updated the example certificates from https://github.com/gramineproject/gramine/tree/master/CI-Examples/ra-tls-secret-prov/certs by -replacing "Common Name" field in the server certificate (i.e., `server2-sha256.crt`) from +replacing the "Common Name" field in the server certificate (i.e., `server2-sha256.crt`) from `localhost` to ``. -In order to create base client and server images for the AKS environment, user can execute +In order to create base client and server images for the AKS environment, user can execute the `base-image-generation-script.sh` script (with sudo). Since both client and server applications will run inside containers in the AKS cluster, and the client application will send its SGX quote to the server for verification, therefore the user needs to graminize the client application. Hence, the @@ -37,7 +37,7 @@ AKS cluster. 1. The `base-image-generation-script.sh` script will create the native Docker server image with the name `aks-secret-prov-server-img:latest`. -2. Push server image to Docker Hub or your preferred registry: +2. Push the server image to Docker Hub or your preferred registry: ```sh $ docker tag aks-secret-prov-server-img:latest \ @@ -45,9 +45,9 @@ AKS cluster. $ docker push /aks-secret-prov-server-img:latest ``` -3. Deploy `aks-secret-prov-server-img:latest` in AKS confidential compute cluster: +3. Deploy `aks-secret-prov-server-img:latest` in the AKS confidential compute cluster: - Reference deployment file: - `gsc/examples/aks-attestation/aks-secret-prov-server-deployment.yaml` + `aks-secret-prov-server-deployment.yaml` ### Creating client image @@ -75,27 +75,27 @@ AKS cluster. 6. Deploy `gsc-aks-secret-prov-client-img:latest` in AKS confidential compute cluster: - Reference deployment file: - `gsc/examples/aks-attestation/aks-secret-prov-client-deployment.yaml` + `aks-secret-prov-client-deployment.yaml` **NOTE**: We tested this example with DCAP driver 1.11 specified in the GSC configuration file. ## Deploying client and server images in AKS Confidential Compute cluster -AKS confidential compute cluster can be created using following +AKS confidential compute cluster can be created using the following [link](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-enclave-nodes-aks-get-started). -Gramine performs out-of-proc mode DCAP quote generation. Out-of-proc mode quote generation requires aesmd -service. To fulfill this requirement, AKS provides +Gramine performs out-of-proc mode DCAP quote generation. Out-of-proc mode quote generation requires +aesmd service. To fulfill this requirement, AKS provides the [sgxquotehelper daemonset](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-nodes-aks-addon#out-of-proc-attestation-for-confidential-workloads) (can be enabled by `--enable-sgxquotehelper` during cluster creation). This feature exposes aesmd service for the container node. The service will internally connect with az-dcap-client to fetch the platform collateral required for quote generation. In this demo, the -`aks-secret-prov-client-deployment.yaml` uses aesmd service exposed by AKS with the help of +`aks-secret-prov-client-deployment.yaml` uses aesmd service exposed by AKS with the help of the sgxquotehelper plugin. -In the ra-tls-secret-prov example, the client will generate out-of-proc mode sgx quote that will be -embedded inside RA-TLS certificate. On receiving the quote, the server will internally verify it -using libsgx-dcap-quote-verify library via az-dcap-client library. +In our example, the client will generate the SGX quote that will be embedded inside the RA-TLS +certificate. On receiving the quote, the server will internally verify it using the +libsgx-dcap-quote-verify library via the az-dcap-client library. ### Deployment diff --git a/examples/aks-attestation/base-image-generation-script.sh b/examples/aks-attestation/base-image-generation-script.sh index 52fbed6d..53a83f1e 100755 --- a/examples/aks-attestation/base-image-generation-script.sh +++ b/examples/aks-attestation/base-image-generation-script.sh @@ -3,8 +3,7 @@ if [ -d "gramine" ]; then echo "\n\n ***** gramine directory exists, proceeding to image generation ***** \n\n" else - chmod u+x gramine_build.sh - ./gramine_build.sh + bash ./gramine_build.sh fi # Create Server image diff --git a/examples/aks-attestation/certs/README b/examples/aks-attestation/certs/README index 1c1ae35f..3498ce9b 100644 --- a/examples/aks-attestation/certs/README +++ b/examples/aks-attestation/certs/README @@ -4,10 +4,10 @@ This directory contains pre-generated example certificates. In particular: in client (attester) so that client can verify the server's certificate. - `server2-sha256.crt` -- RSA SHA256 leaf server certificate in PEM format. Loaded in server (verifier), so it will send it to the client during TLS - handshake. The Common Name field is set to + handshake. The "Common Name" field is set to `ra-tls-server-aks-dns.eastus.cloudapp.azure.com`. - `server2.key` -- RSA private key in PEM format. Loaded in server (verifier). Note: These certificates should not be used in production. Here, user should update the server2-sha256.crt with ``Common Name`` pointing to the DNS - name of the ra-tls-secret-prov server deployed at AKS cluster. + name of the ra-tls-secret-prov server deployed in the AKS cluster. diff --git a/examples/aks-attestation/gramine_build.sh b/examples/aks-attestation/gramine_build.sh index 6f0047a0..0c2295cb 100755 --- a/examples/aks-attestation/gramine_build.sh +++ b/examples/aks-attestation/gramine_build.sh @@ -27,10 +27,10 @@ sudo python3 -B -m pip install 'toml>=0.10' 'meson>=0.55' # Download Gramine git clone https://github.com/gramineproject/gramine.git +cd gramine # Generate Signing Key -cd gramine openssl genrsa -3 -out Pal/src/host/Linux-SGX/signer/enclave-key.pem 3072 # Build Gramine with DCAP enabled mode (assuming in-kernel driver) From e2835cabc97449c9de0a513cfcde00388c02f6eb Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Wed, 1 Dec 2021 18:35:55 +0530 Subject: [PATCH 19/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- examples/aks-attestation/README.md | 4 +++- .../aks-attestation/aks-secret-prov-server.dockerfile | 10 +++++----- .../aks-attestation/base-image-generation-script.sh | 2 +- examples/aks-attestation/gramine_build.sh | 6 ++++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/examples/aks-attestation/README.md b/examples/aks-attestation/README.md index cc6814ce..7fb917af 100644 --- a/examples/aks-attestation/README.md +++ b/examples/aks-attestation/README.md @@ -26,12 +26,14 @@ replacing the "Common Name" field in the server certificate (i.e., `server2-sha2 `localhost` to ``. In order to create base client and server images for the AKS environment, user can execute the -`base-image-generation-script.sh` script (with sudo). Since both client and server applications will +`base-image-generation-script.sh` script. Since both client and server applications will run inside containers in the AKS cluster, and the client application will send its SGX quote to the server for verification, therefore the user needs to graminize the client application. Hence, the following two steps create a native Docker server image and a graminized GSC client image for the AKS cluster. +**NOTE**: This example is Ubuntu-specific (tested version is Ubuntu 18.04). + ### Creating server image 1. The `base-image-generation-script.sh` script will create the native Docker server image with the diff --git a/examples/aks-attestation/aks-secret-prov-server.dockerfile b/examples/aks-attestation/aks-secret-prov-server.dockerfile index 12b57130..73307d22 100644 --- a/examples/aks-attestation/aks-secret-prov-server.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-server.dockerfile @@ -34,11 +34,11 @@ COPY gramine/CI-Examples/ra-tls-secret-prov/secret_prov_server_dcap /usr/local/b RUN mkdir libs -COPY gramine/build/Pal/src/host/Linux-SGX/tools/ra-tls/libsecret_prov_verify_dcap.so libs -COPY gramine/build/Pal/src/host/Linux-SGX/tools/common/libsgx_util.so libs -COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedcrypto_gramine.so.6 libs -COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedtls_gramine.so.13 libs -COPY gramine/build/subprojects/mbedtls-mbedtls-2.26.0/libmbedx509_gramine.so.1 libs +COPY gramine/meson_build_output/lib/x86_64-linux-gnu/libsecret_prov_verify_dcap.so libs +COPY gramine/meson_build_output/lib/x86_64-linux-gnu/libsgx_util.so libs +COPY gramine/meson_build_output/lib/x86_64-linux-gnu/libmbedcrypto_gramine.so.* libs +COPY gramine/meson_build_output/lib/x86_64-linux-gnu/libmbedtls_gramine.so.* libs +COPY gramine/meson_build_output/lib/x86_64-linux-gnu/libmbedx509_gramine.so.* libs ENV LD_LIBRARY_PATH = "${LD_LIBRARY_PATH}:/ra-tls-secret-prov/libs" diff --git a/examples/aks-attestation/base-image-generation-script.sh b/examples/aks-attestation/base-image-generation-script.sh index 53a83f1e..28e38bcb 100755 --- a/examples/aks-attestation/base-image-generation-script.sh +++ b/examples/aks-attestation/base-image-generation-script.sh @@ -1,7 +1,7 @@ # Download and build Gramine directory if [ -d "gramine" ]; then - echo "\n\n ***** gramine directory exists, proceeding to image generation ***** \n\n" + echo "***** gramine directory exists, proceeding to image generation *****" else bash ./gramine_build.sh fi diff --git a/examples/aks-attestation/gramine_build.sh b/examples/aks-attestation/gramine_build.sh index 0c2295cb..4828f915 100755 --- a/examples/aks-attestation/gramine_build.sh +++ b/examples/aks-attestation/gramine_build.sh @@ -28,6 +28,7 @@ sudo python3 -B -m pip install 'toml>=0.10' 'meson>=0.55' git clone https://github.com/gramineproject/gramine.git cd gramine +mkdir -p $PWD/meson_build_output # Generate Signing Key @@ -35,9 +36,10 @@ openssl genrsa -3 -out Pal/src/host/Linux-SGX/signer/enclave-key.pem 3072 # Build Gramine with DCAP enabled mode (assuming in-kernel driver) -meson setup build/ --buildtype=release -Ddirect=enabled -Dsgx=enabled -Ddcap=enabled +meson setup build/ --prefix="$PWD/meson_build_output" --buildtype=release -Ddirect=enabled \ + -Dsgx=enabled -Ddcap=enabled ninja -C build/ -sudo ninja -C build/ install +ninja -C build/ install # Copy dummy server certificate with Common Name as " cd ../ From 5442d9e78a5c70cafcdde54f6d57a673d2488e68 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Wed, 1 Dec 2021 18:39:29 +0530 Subject: [PATCH 20/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- examples/aks-attestation/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/aks-attestation/README.md b/examples/aks-attestation/README.md index 7fb917af..b5934756 100644 --- a/examples/aks-attestation/README.md +++ b/examples/aks-attestation/README.md @@ -32,7 +32,7 @@ server for verification, therefore the user needs to graminize the client applic following two steps create a native Docker server image and a graminized GSC client image for the AKS cluster. -**NOTE**: This example is Ubuntu-specific (tested version is Ubuntu 18.04). +Note: This example is Ubuntu-specific (tested version is Ubuntu 18.04). ### Creating server image @@ -79,7 +79,7 @@ AKS cluster. - Reference deployment file: `aks-secret-prov-client-deployment.yaml` -**NOTE**: We tested this example with DCAP driver 1.11 specified in the GSC configuration file. +Note: We tested this example with DCAP driver 1.11 specified in the GSC configuration file. ## Deploying client and server images in AKS Confidential Compute cluster From bfb5839f0e4ffb9220a0858bf76f159912b5374f Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Thu, 2 Dec 2021 14:45:13 +0530 Subject: [PATCH 21/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- .../aks-attestation/base-image-generation-script.sh | 13 ++++++++++++- examples/aks-attestation/gramine_build.sh | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/examples/aks-attestation/base-image-generation-script.sh b/examples/aks-attestation/base-image-generation-script.sh index 28e38bcb..8ba6dfde 100755 --- a/examples/aks-attestation/base-image-generation-script.sh +++ b/examples/aks-attestation/base-image-generation-script.sh @@ -1,4 +1,6 @@ -# Download and build Gramine directory +# Download and build Gramine directory. We need it to copy ra-tls-secret-prov files and relevant +# libraries into the server and client Dockerfiles. This script also expects Gramine to be installed +# on the system. if [ -d "gramine" ]; then echo "***** gramine directory exists, proceeding to image generation *****" @@ -6,6 +8,15 @@ else bash ./gramine_build.sh fi +# Include Meson build output directory in $PATH +export PATH="$PWD/gramine/meson_build_output/bin:$PATH" + +# Include Meson build output Python dir in $PYTHONPATH, needed by gramine-sgx-get-token +export PYTHONPATH="${PYTHONPATH}:$(find $PWD/gramine/meson_build_output/lib -type d -path '*/site-packages')" + +# Include Meson build output packages dir in $PKG_CONFIG_PATH, contains mbedTLS and util libs +export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$(find $PWD/gramine/meson_build_output/lib -type d -path '*/pkgconfig')" + # Create Server image cd gramine/CI-Examples/ra-tls-secret-prov diff --git a/examples/aks-attestation/gramine_build.sh b/examples/aks-attestation/gramine_build.sh index 4828f915..82c3c130 100755 --- a/examples/aks-attestation/gramine_build.sh +++ b/examples/aks-attestation/gramine_build.sh @@ -28,7 +28,7 @@ sudo python3 -B -m pip install 'toml>=0.10' 'meson>=0.55' git clone https://github.com/gramineproject/gramine.git cd gramine -mkdir -p $PWD/meson_build_output +mkdir -p meson_build_output # Generate Signing Key From 395059dffaa31a0e617cc3bc443b29b031c327af Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Thu, 2 Dec 2021 16:46:49 +0530 Subject: [PATCH 22/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- examples/aks-attestation/base-image-generation-script.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/aks-attestation/base-image-generation-script.sh b/examples/aks-attestation/base-image-generation-script.sh index 8ba6dfde..5e48d324 100755 --- a/examples/aks-attestation/base-image-generation-script.sh +++ b/examples/aks-attestation/base-image-generation-script.sh @@ -1,6 +1,5 @@ -# Download and build Gramine directory. We need it to copy ra-tls-secret-prov files and relevant -# libraries into the server and client Dockerfiles. This script also expects Gramine to be installed -# on the system. +# Download and build Gramine directory. We need it to build and copy ra-tls-secret-prov files and +# relevant libraries into the server and client Dockerfiles. if [ -d "gramine" ]; then echo "***** gramine directory exists, proceeding to image generation *****" @@ -12,10 +11,10 @@ fi export PATH="$PWD/gramine/meson_build_output/bin:$PATH" # Include Meson build output Python dir in $PYTHONPATH, needed by gramine-sgx-get-token -export PYTHONPATH="${PYTHONPATH}:$(find $PWD/gramine/meson_build_output/lib -type d -path '*/site-packages')" +export PYTHONPATH="$(find $PWD/gramine/meson_build_output/lib -type d -path '*/site-packages'):${PYTHONPATH}" # Include Meson build output packages dir in $PKG_CONFIG_PATH, contains mbedTLS and util libs -export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$(find $PWD/gramine/meson_build_output/lib -type d -path '*/pkgconfig')" +export PKG_CONFIG_PATH="$(find $PWD/gramine/meson_build_output/lib -type d -path '*/pkgconfig'):${PKG_CONFIG_PATH}" # Create Server image From 0ae230b75b1809fa429473f8daa13eb8b2dbcdf2 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Fri, 3 Dec 2021 14:07:21 +0530 Subject: [PATCH 23/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- .../aks-attestation/aks-secret-prov-server.dockerfile | 1 + .../aks-attestation/base-image-generation-script.sh | 11 +---------- examples/aks-attestation/gramine_build.sh | 4 ---- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/examples/aks-attestation/aks-secret-prov-server.dockerfile b/examples/aks-attestation/aks-secret-prov-server.dockerfile index 73307d22..6896f389 100644 --- a/examples/aks-attestation/aks-secret-prov-server.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-server.dockerfile @@ -18,6 +18,7 @@ RUN wget https://packages.microsoft.com/ubuntu/18.04/prod/pool/main/a/az-dcap-cl && dpkg -i az-dcap-client_1.10_amd64.deb # Installing DCAP Quote Verification Library + RUN echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' \ > /etc/apt/sources.list.d/intel-sgx.list \ && wget https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \ diff --git a/examples/aks-attestation/base-image-generation-script.sh b/examples/aks-attestation/base-image-generation-script.sh index 5e48d324..60c3bbe4 100755 --- a/examples/aks-attestation/base-image-generation-script.sh +++ b/examples/aks-attestation/base-image-generation-script.sh @@ -1,30 +1,21 @@ -# Download and build Gramine directory. We need it to build and copy ra-tls-secret-prov files and +# Download and build Gramine. We need it to build and copy ra-tls-secret-prov files and # relevant libraries into the server and client Dockerfiles. - if [ -d "gramine" ]; then echo "***** gramine directory exists, proceeding to image generation *****" else bash ./gramine_build.sh fi -# Include Meson build output directory in $PATH -export PATH="$PWD/gramine/meson_build_output/bin:$PATH" - -# Include Meson build output Python dir in $PYTHONPATH, needed by gramine-sgx-get-token -export PYTHONPATH="$(find $PWD/gramine/meson_build_output/lib -type d -path '*/site-packages'):${PYTHONPATH}" - # Include Meson build output packages dir in $PKG_CONFIG_PATH, contains mbedTLS and util libs export PKG_CONFIG_PATH="$(find $PWD/gramine/meson_build_output/lib -type d -path '*/pkgconfig'):${PKG_CONFIG_PATH}" # Create Server image - cd gramine/CI-Examples/ra-tls-secret-prov make clean && make dcap cd ../../../ docker build -f aks-secret-prov-server.dockerfile -t aks-secret-prov-server-img . # Create Client image - cd gramine/CI-Examples/ra-tls-secret-prov make clean && make secret_prov_min_client cd ../../../ diff --git a/examples/aks-attestation/gramine_build.sh b/examples/aks-attestation/gramine_build.sh index 82c3c130..7d582190 100755 --- a/examples/aks-attestation/gramine_build.sh +++ b/examples/aks-attestation/gramine_build.sh @@ -2,7 +2,6 @@ # Please refer to https://gramine.readthedocs.io/en/latest/building.html#id2 for more details. # install Gramine dependencies - sudo apt-get install -y \ autoconf \ bison \ @@ -25,17 +24,14 @@ sudo apt-get install -y \ sudo python3 -B -m pip install 'toml>=0.10' 'meson>=0.55' # Download Gramine - git clone https://github.com/gramineproject/gramine.git cd gramine mkdir -p meson_build_output # Generate Signing Key - openssl genrsa -3 -out Pal/src/host/Linux-SGX/signer/enclave-key.pem 3072 # Build Gramine with DCAP enabled mode (assuming in-kernel driver) - meson setup build/ --prefix="$PWD/meson_build_output" --buildtype=release -Ddirect=enabled \ -Dsgx=enabled -Ddcap=enabled ninja -C build/ From 5c57c98b4294495a2a7c00572e51adba87ced403 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Fri, 3 Dec 2021 14:17:40 +0530 Subject: [PATCH 24/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- examples/aks-attestation/aks-secret-prov-client.dockerfile | 1 - examples/aks-attestation/aks-secret-prov-server.dockerfile | 2 -- 2 files changed, 3 deletions(-) diff --git a/examples/aks-attestation/aks-secret-prov-client.dockerfile b/examples/aks-attestation/aks-secret-prov-client.dockerfile index 3275d8a0..fea5094e 100644 --- a/examples/aks-attestation/aks-secret-prov-client.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-client.dockerfile @@ -8,7 +8,6 @@ RUN apt-get update \ python3 # Installing DCAP libraries - RUN echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' \ > /etc/apt/sources.list.d/intel-sgx.list \ && wget https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \ diff --git a/examples/aks-attestation/aks-secret-prov-server.dockerfile b/examples/aks-attestation/aks-secret-prov-server.dockerfile index 6896f389..fee3590d 100644 --- a/examples/aks-attestation/aks-secret-prov-server.dockerfile +++ b/examples/aks-attestation/aks-secret-prov-server.dockerfile @@ -13,12 +13,10 @@ RUN apt-get update \ # Here, the version of az-dcap-client should be in sync with the az-dcap-client # version used for quote generation. User can replace the below package with the # latest package. - RUN wget https://packages.microsoft.com/ubuntu/18.04/prod/pool/main/a/az-dcap-client/az-dcap-client_1.10_amd64.deb \ && dpkg -i az-dcap-client_1.10_amd64.deb # Installing DCAP Quote Verification Library - RUN echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' \ > /etc/apt/sources.list.d/intel-sgx.list \ && wget https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \ From 06a63d75e5b342d8377ada413e9f596b23368223 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Fri, 3 Dec 2021 16:15:39 +0530 Subject: [PATCH 25/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- examples/aks-attestation/gramine_build.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/aks-attestation/gramine_build.sh b/examples/aks-attestation/gramine_build.sh index 7d582190..ecea0d5a 100755 --- a/examples/aks-attestation/gramine_build.sh +++ b/examples/aks-attestation/gramine_build.sh @@ -31,6 +31,13 @@ mkdir -p meson_build_output # Generate Signing Key openssl genrsa -3 -out Pal/src/host/Linux-SGX/signer/enclave-key.pem 3072 +# Install DCAP dependencies +echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' | \ + sudo tee /etc/apt/sources.list.d/intel-sgx.list +wget https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key +sudo apt-key add intel-sgx-deb.key +sudo apt-get install -y libsgx-dcap-quote-verify-dev + # Build Gramine with DCAP enabled mode (assuming in-kernel driver) meson setup build/ --prefix="$PWD/meson_build_output" --buildtype=release -Ddirect=enabled \ -Dsgx=enabled -Ddcap=enabled From 052f683d9cbd78c757f05b879d700d598dcfa58e Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Mon, 6 Dec 2021 16:26:41 +0530 Subject: [PATCH 26/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- examples/aks-attestation/gramine_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/aks-attestation/gramine_build.sh b/examples/aks-attestation/gramine_build.sh index ecea0d5a..81980c1d 100755 --- a/examples/aks-attestation/gramine_build.sh +++ b/examples/aks-attestation/gramine_build.sh @@ -36,7 +36,7 @@ echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic sudo tee /etc/apt/sources.list.d/intel-sgx.list wget https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key sudo apt-key add intel-sgx-deb.key -sudo apt-get install -y libsgx-dcap-quote-verify-dev +sudo apt-get install --no-install-recommends -y libsgx-urts libsgx-dcap-quote-verify-dev # Build Gramine with DCAP enabled mode (assuming in-kernel driver) meson setup build/ --prefix="$PWD/meson_build_output" --buildtype=release -Ddirect=enabled \ From ad70844bebd80955bec99cd645b7e792cd871520 Mon Sep 17 00:00:00 2001 From: Veena Saini Date: Wed, 8 Dec 2021 19:07:26 +0530 Subject: [PATCH 27/27] fixup! added base server and client image generation script Signed-off-by: Veena Saini --- examples/aks-attestation/README.md | 4 ++-- .../aks-attestation/aks-secret-prov-client-deployment.yaml | 2 +- .../aks-attestation/aks-secret-prov-server-deployment.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/aks-attestation/README.md b/examples/aks-attestation/README.md index b5934756..1fc117ae 100644 --- a/examples/aks-attestation/README.md +++ b/examples/aks-attestation/README.md @@ -58,6 +58,8 @@ Note: This example is Ubuntu-specific (tested version is Ubuntu 18.04). 2. Create the GSC client image: + Note: We tested this example with DCAP driver 1.11 specified in the GSC configuration file. + ```sh $ cd gsc $ cp config.yaml.template config.yaml @@ -79,8 +81,6 @@ Note: This example is Ubuntu-specific (tested version is Ubuntu 18.04). - Reference deployment file: `aks-secret-prov-client-deployment.yaml` -Note: We tested this example with DCAP driver 1.11 specified in the GSC configuration file. - ## Deploying client and server images in AKS Confidential Compute cluster AKS confidential compute cluster can be created using the following diff --git a/examples/aks-attestation/aks-secret-prov-client-deployment.yaml b/examples/aks-attestation/aks-secret-prov-client-deployment.yaml index ac875be2..f2c92792 100644 --- a/examples/aks-attestation/aks-secret-prov-client-deployment.yaml +++ b/examples/aks-attestation/aks-secret-prov-client-deployment.yaml @@ -19,7 +19,7 @@ spec: image: /gsc-aks-secret-prov-client-img:latest env: - name: SECRET_PROVISION_SERVERS - value: ":4433" + value: "ra-tls-server-aks-dns.eastus.cloudapp.azure.com:4433" resources: limits: kubernetes.azure.com/sgx_epc_mem_in_MiB: 10 diff --git a/examples/aks-attestation/aks-secret-prov-server-deployment.yaml b/examples/aks-attestation/aks-secret-prov-server-deployment.yaml index 9a43cbed..10419f03 100644 --- a/examples/aks-attestation/aks-secret-prov-server-deployment.yaml +++ b/examples/aks-attestation/aks-secret-prov-server-deployment.yaml @@ -29,7 +29,7 @@ apiVersion: v1 kind: Service metadata: annotations: - service.beta.kubernetes.io/azure-dns-label-name: + service.beta.kubernetes.io/azure-dns-label-name: ra-tls-server-aks-dns name: ra-tls-secret-prov-server spec: type: LoadBalancer