From a98af27ff7a960ddc15afe0298df81f9f72fc7f1 Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Wed, 2 Oct 2024 13:48:23 +0100 Subject: [PATCH] kbs: Remove oci.kbs version and references Our KBS logic currently checks out the git.kbs version of code, but then edits the deployment config to override the image based on the oci.kbs version. This means that there is an assumption that the oci image of the KBS is compatible with the kustomization, which might not not always be the case and in the "always safe" case where the image is built from that exact version of code (e.g. releases), just means that we have to specify it in two places, so there isn't much advantage to this approach. This PR removes the oci.kbs image and versioning to avoid this potential incompatibility and simplify trustee version updates. Because there is a chance that the trustee team might not keep their kustomizational image up-to-date, the recommendation is that we should ensure that the image version matches the branch we are using, so perform a kustomize set image at the time we clone the repo. As trustee have a separate container registry for development versus release images for reasons, they only automatically push builds to the staged-images registry, which are tagged by the sha of the commit, so we need to convert the KBS_VERSION given into the sha and then replace the image with the staged-images kbs Fixes: #2076 Signed-off-by: stevenhorsman --- .github/workflows/azure-e2e-test.yml | 21 ++----- .github/workflows/e2e_libvirt.yaml | 6 ++ .../libvirt/config_libvirt.sh | 4 -- src/cloud-api-adaptor/test/e2e/README.md | 12 ++-- src/cloud-api-adaptor/test/e2e/main_test.go | 7 --- .../azure/provision_azure.properties | 4 -- .../azure/provision_azure_cli_auth.properties | 2 - .../provisioner/azure/provision_common.go | 2 - .../azure/provision_initializer.go | 4 -- .../provisioner/docker/provision_common.go | 6 -- .../docker/provision_docker.properties | 4 -- .../provisioner/libvirt/provision_common.go | 58 +++++++------------ .../test/provisioner/trustee_kbs.go | 12 +--- .../test/tools/provisioner-cli/main.go | 4 +- src/cloud-api-adaptor/versions.yaml | 3 - 15 files changed, 42 insertions(+), 107 deletions(-) diff --git a/.github/workflows/azure-e2e-test.yml b/.github/workflows/azure-e2e-test.yml index f5a222bee..c68eacda3 100644 --- a/.github/workflows/azure-e2e-test.yml +++ b/.github/workflows/azure-e2e-test.yml @@ -28,12 +28,6 @@ on: caa-image: type: string description: prebuilt caa image - kbs-image-id: - type: string - description: prebuild kbs image - kbs-image-tag: - type: string - description: prebuild kbs image tag jobs: build-caa-container-image: @@ -117,17 +111,10 @@ jobs: echo "TEST_PROVISION_FILE=${{ format(env.TEST_PROVISION_PATH_TEMPLATE, matrix.parameters.id) }}" >> "$GITHUB_ENV" echo "CLUSTER_NAME=${{ format(env.CLUSTER_NAME_TEMPLATE, matrix.parameters.id) }}" >> "$GITHUB_ENV" - - name: Extract provisioner environment - run: | - echo "KBS_IMAGE=$(yq -e '.oci.kbs.registry' versions.yaml)" >> "$GITHUB_ENV" - echo "KBS_IMAGE_TAG=$(yq -e '.oci.kbs.tag' versions.yaml)" >> "$GITHUB_ENV" - - name: Create provisioner file env: AZURE_IMAGE_ID: ${{ inputs.podvm-image-id }} CAA_IMAGE: "${{ github.event.inputs.caa-image || needs.build-caa-container-image.outputs.caa-image }}" - KBS_IMAGE: ${{ github.event.inputs.kbs-image-id || env.KBS_IMAGE }} - KBS_IMAGE_TAG: ${{ github.event.inputs.kbs-image-tag || env.KBS_IMAGE_TAG}} AZURE_INSTANCE_SIZE: ${{ matrix.parameters.machine_type }} run: | echo "Provisioner file for ${{ matrix.parameters.id }}" @@ -142,8 +129,6 @@ jobs: IS_CI_MANAGED_CLUSTER="true" MANAGED_IDENTITY_NAME="${{ secrets.AZURE_MANAGED_IDENTITY_NAME}}" CAA_IMAGE="${CAA_IMAGE}" - KBS_IMAGE="${KBS_IMAGE}" - KBS_IMAGE_TAG="${KBS_IMAGE_TAG}" AZURE_INSTANCE_SIZE="${AZURE_INSTANCE_SIZE}" TAGS="${{ env.TEST_TAGS }}" FEDERATED_CREDENTIAL_NAME="${{ env.CLUSTER_NAME }}" @@ -250,6 +235,12 @@ jobs: git clone https://github.com/confidential-containers/trustee test/trustee pushd test/trustee git checkout "${KBS_VERSION}" + pushd kbs/config/kubernetes/base/ + # Trustee only updates their staging image reliably with sha tags, + # so switch to use that and convert the version to the sha + KBS_SHA=$(gh api repos/confidential-containers/trustee/commits/${KBS_VERSION} -q .sha) + kustomize edit set image kbs-container-image=ghcr.io/confidential-containers/staged-images/kbs:${KBS_SHA} + popd popd - name: Run e2e test diff --git a/.github/workflows/e2e_libvirt.yaml b/.github/workflows/e2e_libvirt.yaml index 31b4b6d51..548f3f65c 100644 --- a/.github/workflows/e2e_libvirt.yaml +++ b/.github/workflows/e2e_libvirt.yaml @@ -105,6 +105,12 @@ jobs: git checkout "${KBS_VERSION}" pushd kbs make CLI_FEATURES=sample_only cli + pushd config/kubernetes/base/ + # Trustee only updates their staging image reliably with sha tags, + # so switch to use that and convert the version to the sha + KBS_SHA=$(gh api repos/confidential-containers/trustee/commits/${KBS_VERSION} -q .sha) + kustomize edit set image kbs-container-image=ghcr.io/confidential-containers/staged-images/kbs:${KBS_SHA} + popd popd # For debugging ls ./target/release diff --git a/src/cloud-api-adaptor/libvirt/config_libvirt.sh b/src/cloud-api-adaptor/libvirt/config_libvirt.sh index fbc6187aa..a3d33aacb 100755 --- a/src/cloud-api-adaptor/libvirt/config_libvirt.sh +++ b/src/cloud-api-adaptor/libvirt/config_libvirt.sh @@ -118,7 +118,3 @@ rm -f libvirt.properties echo "libvirt_uri=\"qemu+ssh://${USER}@${IP}/system?no_verify=1\"" >> libvirt.properties echo "libvirt_ssh_key_file=\"id_rsa\"" >> libvirt.properties echo "CLUSTER_NAME=\"peer-pods\"" >> libvirt.properties -KBS_IMAGE=$(./hack/yq-shim.sh '.oci.kbs.registry' ./versions.yaml) -KBS_IMAGE_TAG=$(./hack/yq-shim.sh '.oci.kbs.tag' ./versions.yaml) -[ -z ${KBS_IMAGE} ] || echo "KBS_IMAGE=\"${KBS_IMAGE}\"" >> libvirt.properties -[ -z ${KBS_IMAGE_TAG} ] || echo "KBS_IMAGE_TAG=\"${KBS_IMAGE_TAG}\"" >> libvirt.properties diff --git a/src/cloud-api-adaptor/test/e2e/README.md b/src/cloud-api-adaptor/test/e2e/README.md index 49ed2e627..6f055beaf 100644 --- a/src/cloud-api-adaptor/test/e2e/README.md +++ b/src/cloud-api-adaptor/test/e2e/README.md @@ -63,8 +63,12 @@ To prepare trustee, execute the following steps: pushd ${cloud-api-adaptor-repo-dir}/src/cloud-api-adaptor/test git clone https://github.com/confidential-containers/trustee.git pushd trustee -git checkout $(../../hack/yq-shim.sh '.git.kbs.reference' ../../versions.yaml) +KBS_VERSION=$(../../hack/yq-shim.sh '.git.kbs.reference' ../../versions.yaml) +git checkout ${KBS_VERSION} pushd kbs +pushd config/kubernetes/base/ +kustomize edit set image kbs-container-image=*:${KBS_VERSION} +popd make CLI_FEATURES=sample_only cli popd popd @@ -81,12 +85,10 @@ popd Then extract the PodVM image and use it following [extracting-the-qcow2-image](../../podvm/README.md#extracting-the-qcow2-image) -To deploy the KBS service and test attestation related cases, export the following variables like: +To deploy the KBS service and test attestation related cases, export the following variable: ```sh export DEPLOY_KBS=yes -export KBS_IMAGE=$(./hack/yq-shim.sh '.oci.kbs.registry' ./versions.yaml) -export KBS_IMAGE_TAG=$(./hack/yq-shim.sh '.oci.kbs.tag' ./versions.yaml) ```` # Running end-to-end tests against pre-configured cluster @@ -256,4 +258,4 @@ func TestCloudProviderCreateSimplePod(t *testing.T) { assert := MyAssert{} DoTestCreateSimplePod(t, assert) } -``` \ No newline at end of file +``` diff --git a/src/cloud-api-adaptor/test/e2e/main_test.go b/src/cloud-api-adaptor/test/e2e/main_test.go index 8363b4711..330b5d591 100644 --- a/src/cloud-api-adaptor/test/e2e/main_test.go +++ b/src/cloud-api-adaptor/test/e2e/main_test.go @@ -5,7 +5,6 @@ package e2e import ( "context" - "fmt" "os" "testing" @@ -123,13 +122,7 @@ func TestMain(m *testing.M) { var err error // Get properties - props := provisioner.GetProperties(ctx, cfg) - if shouldDeployKbs { - if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" { - return ctx, fmt.Errorf("kbs image not provided") - } - } // Set CONTAINER_RUNTIME env variable if present in the properties // Default value is containerd. diff --git a/src/cloud-api-adaptor/test/provisioner/azure/provision_azure.properties b/src/cloud-api-adaptor/test/provisioner/azure/provision_azure.properties index 8099f80cd..8c595e55c 100644 --- a/src/cloud-api-adaptor/test/provisioner/azure/provision_azure.properties +++ b/src/cloud-api-adaptor/test/provisioner/azure/provision_azure.properties @@ -14,10 +14,6 @@ AZURE_CLI_AUTH="false" IS_CI_MANAGED_CLUSTER="false" IS_SELF_MANAGED_CLUSTER="false" -# KBS configs -KBS_IMAGE="${KBS_IMAGE}" -KBS_IMAGE_TAG="${KBS_IMAGE_TAG}" - AZURE_INSTANCE_SIZE="${AZURE_INSTANCE_SIZE}" # either "containerd" or "crio" diff --git a/src/cloud-api-adaptor/test/provisioner/azure/provision_azure_cli_auth.properties b/src/cloud-api-adaptor/test/provisioner/azure/provision_azure_cli_auth.properties index 04c7a4e91..1313d1941 100644 --- a/src/cloud-api-adaptor/test/provisioner/azure/provision_azure_cli_auth.properties +++ b/src/cloud-api-adaptor/test/provisioner/azure/provision_azure_cli_auth.properties @@ -10,6 +10,4 @@ SSH_USERNAME="" AZURE_CLI_AUTH="true" IS_CI_MANAGED_CLUSTER="true" CAA_IMAGE="quay.io/confidential-containers/cloud-api-adaptor:latest" -KBS_IMAGE="${KBS_IMAGE}" -KBS_IMAGE_TAG="${KBS_IMAGE_TAG}" AZURE_INSTANCE_SIZE="${AZURE_INSTANCE_SIZE}" diff --git a/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go b/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go index 2f2d2482a..88180b4d2 100644 --- a/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go @@ -359,8 +359,6 @@ func getPropertiesImpl() map[string]string { "AZURE_IMAGE_ID": AzureProps.ImageID, "AZURE_SUBNET_ID": AzureProps.SubnetID, "AZURE_INSTANCE_SIZE": AzureProps.InstanceSize, - "KBS_IMAGE": AzureProps.KbsImage, - "KBS_IMAGE_TAG": AzureProps.KbsImageTag, "TAGS": AzureProps.Tags, "CONTAINER_RUNTIME": AzureProps.ContainerRuntime, } diff --git a/src/cloud-api-adaptor/test/provisioner/azure/provision_initializer.go b/src/cloud-api-adaptor/test/provisioner/azure/provision_initializer.go index 662aabefb..bbd4d1d43 100644 --- a/src/cloud-api-adaptor/test/provisioner/azure/provision_initializer.go +++ b/src/cloud-api-adaptor/test/provisioner/azure/provision_initializer.go @@ -33,8 +33,6 @@ type AzureProperties struct { IsCIManaged bool CaaImage string IsSelfManaged bool - KbsImage string - KbsImageTag string Tags string InstanceSize string @@ -68,8 +66,6 @@ func initAzureProperties(properties map[string]string) error { SshUserName: properties["SSH_USERNAME"], ManagedIdentityName: properties["MANAGED_IDENTITY_NAME"], CaaImage: properties["CAA_IMAGE"], - KbsImage: properties["KBS_IMAGE"], - KbsImageTag: properties["KBS_IMAGE_TAG"], InstanceSize: properties["AZURE_INSTANCE_SIZE"], Tags: properties["TAGS"], FederatedCredentialName: properties["FEDERATED_CREDENTIAL_NAME"], diff --git a/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go index 064f25120..948d3c8e4 100644 --- a/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go @@ -38,8 +38,6 @@ type DockerProperties struct { PodvmImage string CaaImage string CaaImageTag string - KbsImage string - KbsImageTag string ContainerRuntime string } @@ -55,8 +53,6 @@ func initDockerProperties(properties map[string]string) error { PodvmImage: properties["DOCKER_PODVM_IMAGE"], CaaImage: properties["CAA_IMAGE"], CaaImageTag: properties["CAA_IMAGE_TAG"], - KbsImage: properties["KBS_IMAGE"], - KbsImageTag: properties["KBS_IMAGE_TAG"], ContainerRuntime: properties["CONTAINER_RUNTIME"], } return nil @@ -138,8 +134,6 @@ func (l *DockerProvisioner) GetProperties(ctx context.Context, cfg *envconf.Conf "DOCKER_PODVM_IMAGE": DockerProps.PodvmImage, "CAA_IMAGE": DockerProps.CaaImage, "CAA_IMAGE_TAG": DockerProps.CaaImageTag, - "KBS_IMAGE": DockerProps.KbsImage, - "KBS_IMAGE_TAG": DockerProps.KbsImageTag, "CONTAINER_RUNTIME": DockerProps.ContainerRuntime, } } diff --git a/src/cloud-api-adaptor/test/provisioner/docker/provision_docker.properties b/src/cloud-api-adaptor/test/provisioner/docker/provision_docker.properties index 30f30afbe..d5d5f094d 100644 --- a/src/cloud-api-adaptor/test/provisioner/docker/provision_docker.properties +++ b/src/cloud-api-adaptor/test/provisioner/docker/provision_docker.properties @@ -6,9 +6,5 @@ DOCKER_NETWORK_NAME="kind" CAA_IMAGE="" CAA_IMAGE_TAG="" -# KBS configs -KBS_IMAGE="" -KBS_IMAGE_TAG="" - # either "containerd" or "crio" CONTAINER_RUNTIME="containerd" diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go index 226ec2c3c..5badefa38 100644 --- a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go @@ -21,16 +21,14 @@ import ( // LibvirtProvisioner implements the CloudProvisioner interface for Libvirt. type LibvirtProvisioner struct { - conn *libvirt.Connect // Libvirt connection - network string // Network name - ssh_key_file string // SSH key file used to connect to Libvirt - storage string // Storage pool name - uri string // Libvirt URI - wd string // libvirt's directory path on this repository - volumeName string // Podvm volume name - clusterName string // Cluster name - kbs_image string // KBS Service OCI Image URL - kbs_image_tag string // KBS Service OCI Image Tag + conn *libvirt.Connect // Libvirt connection + network string // Network name + ssh_key_file string // SSH key file used to connect to Libvirt + storage string // Storage pool name + uri string // Libvirt URI + wd string // libvirt's directory path on this repository + volumeName string // Podvm volume name + clusterName string // Cluster name } // LibvirtInstallOverlay implements the InstallOverlay interface @@ -82,28 +80,16 @@ func NewLibvirtProvisioner(properties map[string]string) (pv.CloudProvisioner, e clusterName = properties["cluster_name"] } - kbs_image := "ghcr.io/confidential-containers/key-broker-service" - if properties["KBS_IMAGE"] != "" { - kbs_image = properties["KBS_IMAGE"] - } - - kbs_image_tag := "latest" - if properties["KBS_IMAGE_TAG"] != "" { - kbs_image_tag = properties["KBS_IMAGE_TAG"] - } - // TODO: Check network and storage are not nil? return &LibvirtProvisioner{ - conn: conn, - network: network, - ssh_key_file: ssh_key_file, - storage: storage, - uri: uri, - wd: wd, - volumeName: vol_name, - clusterName: clusterName, - kbs_image: kbs_image, - kbs_image_tag: kbs_image_tag, + conn: conn, + network: network, + ssh_key_file: ssh_key_file, + storage: storage, + uri: uri, + wd: wd, + volumeName: vol_name, + clusterName: clusterName, }, nil } @@ -203,13 +189,11 @@ func (l *LibvirtProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config) func (l *LibvirtProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string { return map[string]string{ - "network": l.network, - "podvm_volume": l.volumeName, - "ssh_key_file": l.ssh_key_file, - "storage": l.storage, - "uri": l.uri, - "KBS_IMAGE": l.kbs_image, - "KBS_IMAGE_TAG": l.kbs_image_tag, + "network": l.network, + "podvm_volume": l.volumeName, + "ssh_key_file": l.ssh_key_file, + "storage": l.storage, + "uri": l.uri, } } diff --git a/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go b/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go index a628b1ffb..ffeb800ac 100644 --- a/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go +++ b/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go @@ -346,17 +346,7 @@ func (lio *KbsInstallOverlay) Delete(ctx context.Context, cfg *envconf.Config) e } func (lio *KbsInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, props map[string]string) error { - var err error - log.Infof("Updating kbs image with %q", props["KBS_IMAGE"]) - if err = lio.overlay.SetKustomizeImage("kbs-container-image", "newName", props["KBS_IMAGE"]); err != nil { - return err - } - - log.Infof("Updating kbs image tag with %q", props["KBS_IMAGE_TAG"]) - if err = lio.overlay.SetKustomizeImage("kbs-container-image", "newTag", props["KBS_IMAGE_TAG"]); err != nil { - return err - } - + // No edits needed here at the moment return nil } diff --git a/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go b/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go index 8348d48a0..c04b89da7 100644 --- a/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go +++ b/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go @@ -91,9 +91,7 @@ func main() { if shouldDeployKbs { log.Info("Deploying kbs") - if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" { - log.Fatal("kbs image not provided") - } + keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"], cfg) if err != nil { log.Fatal(err) diff --git a/src/cloud-api-adaptor/versions.yaml b/src/cloud-api-adaptor/versions.yaml index 81f917f10..5ab140e5f 100644 --- a/src/cloud-api-adaptor/versions.yaml +++ b/src/cloud-api-adaptor/versions.yaml @@ -51,6 +51,3 @@ oci: pause: registry: docker://registry.k8s.io/pause tag: 3.9 - kbs: - registry: ghcr.io/confidential-containers/key-broker-service - tag: built-in-as-v0.10.1