CI: run e2e tests for libvirt with CRI-O

.github/workflows/e2e_libvirt.yaml

@@ -23,6 +23,11 @@ on:
         description: Git ref to checkout the cloud-api-adaptor repository. Defaults to main.
         required: false
         type: string
+      container_runtime:
+        default: 'containerd'
+        description: Name of the container runtime. Either containerd or crio.
+        required: false
+        type: string
 
 env:
   CLOUD_PROVIDER: libvirt
@@ -35,6 +40,8 @@ defaults:
 jobs:
   test:
     runs-on: az-ubuntu-2204
+    # TODO: remove this when the crio job gets stable
+    continue-on-error: ${{ inputs.container_runtime == 'crio' && true || false }}
     steps:
       - name: Checkout Code
         uses: actions/checkout@v3
@@ -86,6 +93,7 @@ jobs:
         run: |
           ./libvirt/config_libvirt.sh
           echo "CAA_IMAGE=\"${{ inputs.caa_image }}\"" >> libvirt.properties
+          echo "container_runtime=\"${{ inputs.container_runtime }}\"" >> libvirt.properties
           # For debugging
           cat libvirt.properties
 
@@ -117,6 +125,7 @@ jobs:
           REGISTRY_CREDENTIAL_ENCODED: ${{ secrets.REGISTRY_CREDENTIAL_ENCODED }}
         run: |
           export CLOUD_PROVIDER=libvirt
+          export CONTAINER_RUNTIME="${{ inputs.container_runtime }}"
           export DEPLOY_KBS=true
           export TEST_PROVISION="yes"
           export TEST_TEARDOWN="no"

.github/workflows/e2e_run_all.yaml

@@ -159,6 +159,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        container_runtime:
+          - containerd
+          - crio
         os:
           - ubuntu
         provider:
@@ -168,6 +171,7 @@ jobs:
     uses: ./.github/workflows/e2e_libvirt.yaml
     with:
       caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-dev
+      container_runtime: ${{ matrix.container_runtime }}
       podvm_image: ${{ inputs.registry }}/podvm-${{ matrix.provider }}-${{ matrix.os }}-${{ matrix.arch }}:${{ inputs.podvm_image_tag }}
       install_directory_artifact: install_directory
       git_ref: ${{ inputs.git_ref }}

src/cloud-api-adaptor/libvirt/kcli_cluster.sh

@@ -15,6 +15,7 @@ CLUSTER_NAME="${CLUSTER_NAME:-peer-pods}"
 CLUSTER_IMAGE="${CLUSTER_IMAGE:-ubuntu2204}"
 CLUSTER_VERSION="${CLUSTER_VERSION:-1.30.0}"
 CLUSTER_WORKERS="${CLUSTER_WORKERS:-1}"
+CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-containerd}" # Either "containerd" or "crio"
 LIBVIRT_NETWORK="${LIBVIRT_NETWORK:-default}"
 LIBVIRT_POOL="${LIBVIRT_POOL:-default}"
 
@@ -53,7 +54,8 @@ create () {
 		-P sdn=flannel \
 		-P nfs=false \
 		-P disk_size=$CLUSTER_DISK_SIZE \
-		-P version=$CLUSTER_VERSION"
+		-P version=$CLUSTER_VERSION \
+		-P engine=$CONTAINER_RUNTIME"
 	# The autolabeller and multus images do not support s390x arch yet
 	# disable them for s390x cluster
 	if [[ ${TARGET_ARCH} == "s390x" ]]; then
@@ -120,7 +122,8 @@ usage () {
 	          CLUSTER_VERSION         (default "${CLUSTER_VERSION}")
 	          LIBVIRT_NETWORK         (default "${LIBVIRT_NETWORK}")
 	          LIBVIRT_POOL            (default "${LIBVIRT_POOL}")
-	          CLUSTER_WORKERS         (default "${CLUSTER_WORKERS}").
+	          CLUSTER_WORKERS         (default "${CLUSTER_WORKERS}")
+	          CONTAINER_RUNTIME       (default "${CONTAINER_RUNTIME}")
 	delete    Delete the cluster. Specify the cluster name with
 	          CLUSTER_NAME (default "${CLUSTER_NAME}").
 	EOF

src/cloud-api-adaptor/test/e2e/README.md

@@ -150,6 +150,7 @@ Use the properties on the table below for Libvirt:
 
 |Property|Description|Default|
 |---|---|---|
+|container_runtime|Test cluster configured container runtime. Either **containerd** or **crio** |containerd|
 |libvirt_network|Libvirt Network|"default"|
 |libvirt_storage|Libvirt storage pool|"default"|
 |libvirt_vol_name|Volume name|"podvm-base.qcow2"|

src/cloud-api-adaptor/test/e2e/common_suite.go

@@ -632,8 +632,10 @@ func DoTestRestrictivePolicyBlocksExec(t *testing.T, e env.Environment, assert C
 			Command:       []string{"ls"},
 			ContainerName: pod.Spec.Containers[0].Name,
 			TestErrorFn: func(err error) bool {
-				if strings.Contains(err.Error(), "failed to exec in container") && strings.Contains(err.Error(), "ExecProcessRequest is blocked by policy") {
-					t.Logf("Exec process was blocked %s", err.Error())
+				if (strings.Contains(err.Error(), "failed to exec in container") || // containerd
+					strings.Contains(err.Error(), "error executing command in container")) && // cri-o
+					strings.Contains(err.Error(), "ExecProcessRequest is blocked by policy") {
+					t.Logf("Exec process was blocked: %s", err.Error())
 					return true
 				} else {
 					t.Errorf("Exec process was allowed: %s", err.Error())

src/cloud-api-adaptor/test/e2e/libvirt_test.go

@@ -30,6 +30,9 @@ func TestLibvirtCreatePodWithSecret(t *testing.T) {
 
 func TestLibvirtCreatePeerPodContainerWithExternalIPAccess(t *testing.T) {
 	SkipTestOnCI(t)
+	if isTestOnCrio() {
+		t.Skip("Fails with CRI-O (confidential-containers/cloud-api-adaptor#2100)")
+	}
 	assert := LibvirtAssert{}
 	DoTestCreatePeerPodContainerWithExternalIPAccess(t, testEnv, assert)
 
@@ -101,13 +104,19 @@ func TestLibvirtDeletePod(t *testing.T) {
 func TestLibvirtPodToServiceCommunication(t *testing.T) {
 	// This test is causing issues on CI with instability, so skip until we can resolve this.
 	SkipTestOnCI(t)
+	if isTestOnCrio() {
+		t.Skip("Fails with CRI-O (confidential-containers/cloud-api-adaptor#2100)")
+	}
 	assert := LibvirtAssert{}
 	DoTestPodToServiceCommunication(t, testEnv, assert)
 }
 
 func TestLibvirtPodsMTLSCommunication(t *testing.T) {
 	// This test is causing issues on CI with instability, so skip until we can resolve this.
 	SkipTestOnCI(t)
+	if isTestOnCrio() {
+		t.Skip("Fails with CRI-O (confidential-containers/cloud-api-adaptor#2100)")
+	}
 	assert := LibvirtAssert{}
 	DoTestPodsMTLSCommunication(t, testEnv, assert)
 }

src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go

@@ -21,16 +21,17 @@ import (
 
 // LibvirtProvisioner implements the CloudProvisioner interface for Libvirt.
 type LibvirtProvisioner struct {
-	conn          *libvirt.Connect // Libvirt connection
-	network       string           // Network name
-	ssh_key_file  string           // SSH key file used to connect to Libvirt
-	storage       string           // Storage pool name
-	uri           string           // Libvirt URI
-	wd            string           // libvirt's directory path on this repository
-	volumeName    string           // Podvm volume name
-	clusterName   string           // Cluster name
-	kbs_image     string           // KBS Service OCI Image URL
-	kbs_image_tag string           // KBS Service OCI Image Tag
+	conn             *libvirt.Connect // Libvirt connection
+	containerRuntime string           // Name of the container runtime
+	network          string           // Network name
+	ssh_key_file     string           // SSH key file used to connect to Libvirt
+	storage          string           // Storage pool name
+	uri              string           // Libvirt URI
+	wd               string           // libvirt's directory path on this repository
+	volumeName       string           // Podvm volume name
+	clusterName      string           // Cluster name
+	kbs_image        string           // KBS Service OCI Image URL
+	kbs_image_tag    string           // KBS Service OCI Image Tag
 }
 
 // LibvirtInstallOverlay implements the InstallOverlay interface
@@ -94,16 +95,17 @@ func NewLibvirtProvisioner(properties map[string]string) (pv.CloudProvisioner, e
 
 	// TODO: Check network and storage are not nil?
 	return &LibvirtProvisioner{
-		conn:          conn,
-		network:       network,
-		ssh_key_file:  ssh_key_file,
-		storage:       storage,
-		uri:           uri,
-		wd:            wd,
-		volumeName:    vol_name,
-		clusterName:   clusterName,
-		kbs_image:     kbs_image,
-		kbs_image_tag: kbs_image_tag,
+		conn:             conn,
+		containerRuntime: properties["container_runtime"],
+		network:          network,
+		ssh_key_file:     ssh_key_file,
+		storage:          storage,
+		uri:              uri,
+		wd:               wd,
+		volumeName:       vol_name,
+		clusterName:      clusterName,
+		kbs_image:        kbs_image,
+		kbs_image_tag:    kbs_image_tag,
 	}, nil
 }
 
@@ -203,13 +205,14 @@ func (l *LibvirtProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config)
 
 func (l *LibvirtProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string {
 	return map[string]string{
-		"network":       l.network,
-		"podvm_volume":  l.volumeName,
-		"ssh_key_file":  l.ssh_key_file,
-		"storage":       l.storage,
-		"uri":           l.uri,
-		"KBS_IMAGE":     l.kbs_image,
-		"KBS_IMAGE_TAG": l.kbs_image_tag,
+		"CONTAINER_RUNTIME": l.containerRuntime,
+		"network":           l.network,
+		"podvm_volume":      l.volumeName,
+		"ssh_key_file":      l.ssh_key_file,
+		"storage":           l.storage,
+		"uri":               l.uri,
+		"KBS_IMAGE":         l.kbs_image,
+		"KBS_IMAGE_TAG":     l.kbs_image_tag,
 	}
 }
 

src/cloud-api-adaptor/versions.yaml

@@ -24,7 +24,7 @@ tools:
   rust: 1.75.0
   protoc: 3.15.0
   packer: v1.9.4
-  kcli: 99.0.202407031308
+  kcli: 99.0.202408152044
   iptables-wrapper: v0.0.0-20240819165702-06cad2ec6cb5
   oras: 1.2.0
 # Referenced Git repositories