diff --git a/.github/workflows/e2e_libvirt.yaml b/.github/workflows/e2e_libvirt.yaml index 31b4b6d51..234d8f744 100644 --- a/.github/workflows/e2e_libvirt.yaml +++ b/.github/workflows/e2e_libvirt.yaml @@ -23,6 +23,11 @@ on: description: Git ref to checkout the cloud-api-adaptor repository. Defaults to main. required: false type: string + container_runtime: + default: 'containerd' + description: Name of the container runtime. Either containerd or crio. + required: false + type: string env: CLOUD_PROVIDER: libvirt @@ -35,6 +40,8 @@ defaults: jobs: test: runs-on: az-ubuntu-2204 + # TODO: remove this when the crio job gets stable + continue-on-error: ${{ inputs.container_runtime == 'crio' && true || false }} steps: - name: Checkout Code uses: actions/checkout@v3 @@ -86,6 +93,7 @@ jobs: run: | ./libvirt/config_libvirt.sh echo "CAA_IMAGE=\"${{ inputs.caa_image }}\"" >> libvirt.properties + echo "container_runtime=\"${{ inputs.container_runtime }}\"" >> libvirt.properties # For debugging cat libvirt.properties @@ -117,6 +125,7 @@ jobs: REGISTRY_CREDENTIAL_ENCODED: ${{ secrets.REGISTRY_CREDENTIAL_ENCODED }} run: | export CLOUD_PROVIDER=libvirt + export CONTAINER_RUNTIME="${{ inputs.container_runtime }}" export DEPLOY_KBS=true export TEST_PROVISION="yes" export TEST_TEARDOWN="no" diff --git a/.github/workflows/e2e_run_all.yaml b/.github/workflows/e2e_run_all.yaml index 6bdb036b2..c111c5a2e 100644 --- a/.github/workflows/e2e_run_all.yaml +++ b/.github/workflows/e2e_run_all.yaml @@ -159,6 +159,9 @@ jobs: strategy: fail-fast: false matrix: + container_runtime: + - containerd + - crio os: - ubuntu provider: @@ -168,6 +171,7 @@ jobs: uses: ./.github/workflows/e2e_libvirt.yaml with: caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-dev + container_runtime: ${{ matrix.container_runtime }} podvm_image: ${{ inputs.registry }}/podvm-${{ matrix.provider }}-${{ matrix.os }}-${{ matrix.arch }}:${{ inputs.podvm_image_tag }} install_directory_artifact: install_directory git_ref: ${{ inputs.git_ref }} diff --git a/src/cloud-api-adaptor/libvirt/kcli_cluster.sh b/src/cloud-api-adaptor/libvirt/kcli_cluster.sh index 8242bf796..1c5656f3e 100755 --- a/src/cloud-api-adaptor/libvirt/kcli_cluster.sh +++ b/src/cloud-api-adaptor/libvirt/kcli_cluster.sh @@ -15,6 +15,7 @@ CLUSTER_NAME="${CLUSTER_NAME:-peer-pods}" CLUSTER_IMAGE="${CLUSTER_IMAGE:-ubuntu2204}" CLUSTER_VERSION="${CLUSTER_VERSION:-1.30.0}" CLUSTER_WORKERS="${CLUSTER_WORKERS:-1}" +CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-containerd}" # Either "containerd" or "crio" LIBVIRT_NETWORK="${LIBVIRT_NETWORK:-default}" LIBVIRT_POOL="${LIBVIRT_POOL:-default}" @@ -53,7 +54,8 @@ create () { -P sdn=flannel \ -P nfs=false \ -P disk_size=$CLUSTER_DISK_SIZE \ - -P version=$CLUSTER_VERSION" + -P version=$CLUSTER_VERSION \ + -P engine=$CONTAINER_RUNTIME" # The autolabeller and multus images do not support s390x arch yet # disable them for s390x cluster if [[ ${TARGET_ARCH} == "s390x" ]]; then @@ -120,7 +122,8 @@ usage () { CLUSTER_VERSION (default "${CLUSTER_VERSION}") LIBVIRT_NETWORK (default "${LIBVIRT_NETWORK}") LIBVIRT_POOL (default "${LIBVIRT_POOL}") - CLUSTER_WORKERS (default "${CLUSTER_WORKERS}"). + CLUSTER_WORKERS (default "${CLUSTER_WORKERS}") + CONTAINER_RUNTIME (default "${CONTAINER_RUNTIME}") delete Delete the cluster. Specify the cluster name with CLUSTER_NAME (default "${CLUSTER_NAME}"). EOF diff --git a/src/cloud-api-adaptor/test/e2e/README.md b/src/cloud-api-adaptor/test/e2e/README.md index 49ed2e627..3bfeaf294 100644 --- a/src/cloud-api-adaptor/test/e2e/README.md +++ b/src/cloud-api-adaptor/test/e2e/README.md @@ -150,6 +150,7 @@ Use the properties on the table below for Libvirt: |Property|Description|Default| |---|---|---| +|container_runtime|Test cluster configured container runtime. Either **containerd** or **crio** |containerd| |libvirt_network|Libvirt Network|"default"| |libvirt_storage|Libvirt storage pool|"default"| |libvirt_vol_name|Volume name|"podvm-base.qcow2"| diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go index c575c8103..fdb969b05 100644 --- a/src/cloud-api-adaptor/test/e2e/common_suite.go +++ b/src/cloud-api-adaptor/test/e2e/common_suite.go @@ -632,8 +632,10 @@ func DoTestRestrictivePolicyBlocksExec(t *testing.T, e env.Environment, assert C Command: []string{"ls"}, ContainerName: pod.Spec.Containers[0].Name, TestErrorFn: func(err error) bool { - if strings.Contains(err.Error(), "failed to exec in container") && strings.Contains(err.Error(), "ExecProcessRequest is blocked by policy") { - t.Logf("Exec process was blocked %s", err.Error()) + if (strings.Contains(err.Error(), "failed to exec in container") || // containerd + strings.Contains(err.Error(), "error executing command in container")) && // cri-o + strings.Contains(err.Error(), "ExecProcessRequest is blocked by policy") { + t.Logf("Exec process was blocked: %s", err.Error()) return true } else { t.Errorf("Exec process was allowed: %s", err.Error()) diff --git a/src/cloud-api-adaptor/test/e2e/libvirt_test.go b/src/cloud-api-adaptor/test/e2e/libvirt_test.go index 0c6e17f31..9c99187f1 100644 --- a/src/cloud-api-adaptor/test/e2e/libvirt_test.go +++ b/src/cloud-api-adaptor/test/e2e/libvirt_test.go @@ -30,6 +30,9 @@ func TestLibvirtCreatePodWithSecret(t *testing.T) { func TestLibvirtCreatePeerPodContainerWithExternalIPAccess(t *testing.T) { SkipTestOnCI(t) + if isTestOnCrio() { + t.Skip("Fails with CRI-O (confidential-containers/cloud-api-adaptor#2100)") + } assert := LibvirtAssert{} DoTestCreatePeerPodContainerWithExternalIPAccess(t, testEnv, assert) @@ -101,6 +104,9 @@ func TestLibvirtDeletePod(t *testing.T) { func TestLibvirtPodToServiceCommunication(t *testing.T) { // This test is causing issues on CI with instability, so skip until we can resolve this. SkipTestOnCI(t) + if isTestOnCrio() { + t.Skip("Fails with CRI-O (confidential-containers/cloud-api-adaptor#2100)") + } assert := LibvirtAssert{} DoTestPodToServiceCommunication(t, testEnv, assert) } @@ -108,6 +114,9 @@ func TestLibvirtPodToServiceCommunication(t *testing.T) { func TestLibvirtPodsMTLSCommunication(t *testing.T) { // This test is causing issues on CI with instability, so skip until we can resolve this. SkipTestOnCI(t) + if isTestOnCrio() { + t.Skip("Fails with CRI-O (confidential-containers/cloud-api-adaptor#2100)") + } assert := LibvirtAssert{} DoTestPodsMTLSCommunication(t, testEnv, assert) } diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go index 226ec2c3c..0de8c9796 100644 --- a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go @@ -21,16 +21,17 @@ import ( // LibvirtProvisioner implements the CloudProvisioner interface for Libvirt. type LibvirtProvisioner struct { - conn *libvirt.Connect // Libvirt connection - network string // Network name - ssh_key_file string // SSH key file used to connect to Libvirt - storage string // Storage pool name - uri string // Libvirt URI - wd string // libvirt's directory path on this repository - volumeName string // Podvm volume name - clusterName string // Cluster name - kbs_image string // KBS Service OCI Image URL - kbs_image_tag string // KBS Service OCI Image Tag + conn *libvirt.Connect // Libvirt connection + containerRuntime string // Name of the container runtime + network string // Network name + ssh_key_file string // SSH key file used to connect to Libvirt + storage string // Storage pool name + uri string // Libvirt URI + wd string // libvirt's directory path on this repository + volumeName string // Podvm volume name + clusterName string // Cluster name + kbs_image string // KBS Service OCI Image URL + kbs_image_tag string // KBS Service OCI Image Tag } // LibvirtInstallOverlay implements the InstallOverlay interface @@ -94,16 +95,17 @@ func NewLibvirtProvisioner(properties map[string]string) (pv.CloudProvisioner, e // TODO: Check network and storage are not nil? return &LibvirtProvisioner{ - conn: conn, - network: network, - ssh_key_file: ssh_key_file, - storage: storage, - uri: uri, - wd: wd, - volumeName: vol_name, - clusterName: clusterName, - kbs_image: kbs_image, - kbs_image_tag: kbs_image_tag, + conn: conn, + containerRuntime: properties["container_runtime"], + network: network, + ssh_key_file: ssh_key_file, + storage: storage, + uri: uri, + wd: wd, + volumeName: vol_name, + clusterName: clusterName, + kbs_image: kbs_image, + kbs_image_tag: kbs_image_tag, }, nil } @@ -203,13 +205,14 @@ func (l *LibvirtProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config) func (l *LibvirtProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string { return map[string]string{ - "network": l.network, - "podvm_volume": l.volumeName, - "ssh_key_file": l.ssh_key_file, - "storage": l.storage, - "uri": l.uri, - "KBS_IMAGE": l.kbs_image, - "KBS_IMAGE_TAG": l.kbs_image_tag, + "CONTAINER_RUNTIME": l.containerRuntime, + "network": l.network, + "podvm_volume": l.volumeName, + "ssh_key_file": l.ssh_key_file, + "storage": l.storage, + "uri": l.uri, + "KBS_IMAGE": l.kbs_image, + "KBS_IMAGE_TAG": l.kbs_image_tag, } } diff --git a/src/cloud-api-adaptor/versions.yaml b/src/cloud-api-adaptor/versions.yaml index 81f917f10..b8cf07827 100644 --- a/src/cloud-api-adaptor/versions.yaml +++ b/src/cloud-api-adaptor/versions.yaml @@ -24,7 +24,7 @@ tools: rust: 1.75.0 protoc: 3.15.0 packer: v1.9.4 - kcli: 99.0.202407031308 + kcli: 99.0.202408152044 iptables-wrapper: v0.0.0-20240819165702-06cad2ec6cb5 oras: 1.2.0 # Referenced Git repositories