chore(e2e): fixes for more stability

- Ignore failed to watch, fail to elect, fail to create vmip errors in virtualization-controller log.
- Use Eventually to test IP in cilium agents.
- Fix handling kubectl errors in SaveResourcesForTest.
- Wait for snapshots readiness before checking for unfrozen filesystems.

Signed-off-by: Ivan Mikheykin <[email protected]>

Author: diafour

.github/workflows/dev_module_build.yml

@@ -625,7 +625,7 @@ jobs:
         with:
           name: resources_from_failed_tests
           retention-days: 2
-          path: /tmp/e2e_failed__*
+          path: ${{ runner.temp }}/e2e_failed__*
           if-no-files-found: ignore
 
       - name: Cleanup E2E resources on cancel

tests/e2e/default_config.yaml

@@ -51,12 +51,18 @@ logFilter:
   - "lastTransitionTime: Required value" # Err.
   - "virtualmachineipaddressleases.virtualization.deckhouse.io "
   - "Forbidden: no new finalizers can be added if the object is being deleted, found new finalizers"
+  - "Failed to watch" # error if virtualization-controller restarts during tests. "msg": "Failed to watch", "err": "Get \"http://127.0.0.1:23915/apis/virtualization.deckhouse.io/v1alpha2/virtualmachinerestores?allowWatchBookmarks=true\u0026resourceVersion=709816257\u0026timeoutSeconds=310\u0026watch=true\": context canceled"
+  - "leader election lost"
 regexpLogFilter:
   - "failed to detach: .* not found" # "err" "failed to detach: virtualmachine.kubevirt.io \"head-497d17b-vm-automatic-with-hotplug\" not found",
   - "error patching .* not found" # "err" "error patching *** virtualimages.virtualization.deckhouse.io \"head-497d17b-vi-pvc-oref-vi-oref-vd\" not found",
   - "IP address .* is not among addresses assigned to 'default' network interface .*" # "msg": "IP address (10.66.10.61) is not among addresses assigned to 'default' network interface (10.66.10.60)"
   - "failed to get vmSnapshot: VirtualMachineSnapshot\\.virtualization\\.deckhouse.io .* not found" # "msg": "failed to get vmSnapshot: VirtualMachineSnapshot.virtualization.deckhouse.io \"main-to-pr14969-ynv-0-ef17ba-20250908-142437\" not found"
   - "failed to sync virtual disk data source objectref: start immediate: internalvirtualizationdatavolumes.cdi.internal.virtualization.deckhouse.io .* is forbidden: unable to create new content in namespace .* because it is being terminated" # "err": "failed to sync virtual disk data source objectref: start immediate: internalvirtualizationdatavolumes.cdi.internal.virtualization.deckhouse.io \"vd-head-b3d8865-vd-root-migration-bios-d77ea313-f469-463d-a71b-00c89ca542ab\" is forbidden: unable to create new content in namespace head-b3d8865-end-to-end-vm-migration because it is being terminated"
+  - "Failed to update lock optimistically:.*leases.*leader-election-helper.*" # error during virtualization-controller lifecycle: attempt to reacquire leader election. "msg": "Failed to update lock optimistically: Put \"http://127.0.0.1:23915/apis/coordination.k8s.io/v1/namespaces/d8-virtualization/leases/d8-virt-operator-leader-election-helper?timeout=5s\": context deadline exceeded (Client.Timeout exceeded while awaiting headers), falling back to slow path"
+  - "Failed to update lock: .* leases.*leader-election-helper.*" # "msg": "ock: Operation cannot be fulfilled on leases.coordination.k8s.io \"d8-virt-operator-leader-election-helper\": the object has been modified; please apply your changes to the latest version and try again",
+  - "failed to create VirtualMachineIPAddress .* the specified IP address .* has already been allocated and has not been released" #     "err": "failed to create VirtualMachineIPAddress \"head-5d2c558-vm-restore-safe-tfv4w\": admission webhook \"vmip.virtualization-controller.validate.d8-virtualization\" denied the request: the VirtualMachineIPAddress cannot be created: the specified IP address 10.66.10.4 has already been allocated and has not been released"
+  - "error retrieving resource lock .*leader-election-helper" #  "msg": "error retrieving resource lock d8-virtualization/d8-virt-operator-leader-election-helper: context deadline exceeded",
 
 cleanupResources:
   - clustervirtualimages.virtualization.deckhouse.io

tests/e2e/network/cilium_agents.go

@@ -62,7 +62,7 @@ func CheckCilliumAgents(ctx context.Context, kubectl kc.Kubectl, vmName, vmNames
 			}
 
 			if !found {
-				return fmt.Errorf("failed: cilium agent %s for VM's node %s", pod.Name, nodeName)
+				return fmt.Errorf("failed: not found cilium agent %s for VM's node %s", pod.Name, nodeName)
 			}
 		} else {
 			// For pods on different nodes
@@ -72,7 +72,7 @@ func CheckCilliumAgents(ctx context.Context, kubectl kc.Kubectl, vmName, vmNames
 			}
 
 			if !found {
-				return fmt.Errorf("failed: cilium agent %s for node %s", pod.Name, pod.Spec.NodeName)
+				return fmt.Errorf("failed: not found cilium agent %s for node %s", pod.Name, pod.Spec.NodeName)
 			}
 		}
 	}

tests/e2e/util_test.go

@@ -734,6 +734,10 @@ func IsContainerRestarted(podName, containerName, namespace string, startedAt me
 	return false, fmt.Errorf("failed to compare the `startedAt` field before and after the tests ran: %s", podName)
 }
 
+// SaveTestResources dump some resources that may help in further diagnostic.
+//
+// NOTE: This method is called in AfterEach for failed specs only. Avoid to use Expect,
+// as it fails without reporting. Better use GinkgoWriter to report errors at this point.
 func SaveTestResources(labels map[string]string, additional string) {
 	replacer := strings.NewReplacer(
 		" ", "_",
@@ -743,16 +747,35 @@ func SaveTestResources(labels map[string]string, additional string) {
 		"(", "_",
 		")", "_",
 		"|", "_",
+		"`", "",
+		"'", "",
 	)
 	additional = replacer.Replace(strings.ToLower(additional))
 
-	str := fmt.Sprintf("/tmp/e2e_failed__%s__%s.yaml", labels["testcase"], additional)
+	tmpDir := os.Getenv("RUNNER_TEMP")
+	if tmpDir == "" {
+		tmpDir = "/tmp"
+	}
+	resFileName := fmt.Sprintf("%s/e2e_failed__%s__%s.yaml", tmpDir, labels["testcase"], additional)
+	errorFileName := fmt.Sprintf("%s/e2e_failed__%s__%s_error.txt", tmpDir, labels["testcase"], additional)
 
-	cmdr := kubectl.Get("virtualization,intvirt -A", kc.GetOptions{Output: "yaml", Labels: labels})
-	Expect(cmdr.Error()).NotTo(HaveOccurred(), "cmd: %s\nstderr: %s", cmdr.GetCmd(), cmdr.StdErr())
+	cmdr := kubectl.Get("virtualization,intvirt,po -A", kc.GetOptions{Output: "yaml", Labels: labels})
+	if cmdr.Error() != nil {
+		errReport := fmt.Sprintf("cmd: %s\nerror: %s\nstderr: %s\n", cmdr.GetCmd(), cmdr.Error(), cmdr.StdErr())
+		GinkgoWriter.Printf("Get resources error:\n%s\n", errReport)
+		err := os.WriteFile(errorFileName, []byte(errReport), 0o644)
+		if err != nil {
+			GinkgoWriter.Printf("Save error to file '%s' failed: %s\n", errorFileName, err)
+		}
+	}
 
-	err := os.WriteFile(str, cmdr.StdOutBytes(), 0o644)
-	Expect(err).NotTo(HaveOccurred(), "cmd: %s\nstderr: %s", cmdr.GetCmd(), cmdr.StdErr())
+	// Stdout may present even if error is occurred.
+	if len(cmdr.StdOutBytes()) > 0 {
+		err := os.WriteFile(resFileName, cmdr.StdOutBytes(), 0o644)
+		if err != nil {
+			GinkgoWriter.Printf("Save resources to file '%s' failed: %s\n", errorFileName, err)
+		}
+	}
 }
 
 type Watcher interface {

tests/e2e/vd_snapshots_test.go

@@ -264,7 +264,7 @@ var _ = Describe("VirtualDiskSnapshots", ginkgoutil.CommonE2ETestDecorators(), f
 							}(i)
 						}
 						wg.Wait()
-						Expect(errs).To(BeEmpty(), "concurrent snapshotting error")
+						Expect(errs).To(BeEmpty(), "should not face concurrent snapshotting error")
 
 						Eventually(func() error {
 							frozen, err := CheckFileSystemFrozen(vm.Name, ns)
@@ -282,6 +282,28 @@ var _ = Describe("VirtualDiskSnapshots", ginkgoutil.CommonE2ETestDecorators(), f
 			}
 		})
 
+		It("checks snapshots", func() {
+			By("Snapshots should be `Ready`")
+			labels := make(map[string]string)
+			maps.Copy(labels, attachedVirtualDiskLabel)
+			maps.Copy(labels, testCaseLabel)
+
+			Eventually(func() error {
+				vdSnapshots := GetVirtualDiskSnapshots(ns, labels)
+				for _, snapshot := range vdSnapshots.Items {
+					if snapshot.Status.Phase == virtv2.VirtualDiskSnapshotPhaseReady || snapshot.DeletionTimestamp != nil {
+						continue
+					}
+					return errors.New("still wait for all snapshots either in ready or in deletion state")
+				}
+				return nil
+			}).WithTimeout(
+				LongWaitDuration,
+			).WithPolling(
+				Interval,
+			).Should(Succeed(), "all snapshots should be in ready state after creation")
+		})
+
 		// TODO: It is a known issue that disk snapshots are not always created consistently. To prevent this error from causing noise during testing, we disabled this check. It will need to be re-enabled once the consistency issue is fixed.
 		// It("checks snapshots of attached VDs", func() {
 		// 	By(fmt.Sprintf("Snapshots should be in %s phase", PhaseReady))
@@ -319,7 +341,7 @@ var _ = Describe("VirtualDiskSnapshots", ginkgoutil.CommonE2ETestDecorators(), f
 						return nil
 					}
 					if frozen {
-						return fmt.Errorf("the filesystem of the virtual machine %s/%s is frozen", vm.Namespace, vm.Name)
+						return fmt.Errorf("the filesystem of the virtual machine %s/%s is still frozen", vm.Namespace, vm.Name)
 					}
 					return nil
 				}).WithTimeout(
@@ -384,6 +406,18 @@ func CreateVirtualDiskSnapshot(vdName, snapshotName, namespace string, requiredC
 	return nil
 }
 
+func GetVirtualDiskSnapshots(namespace string, labels map[string]string) virtv2.VirtualDiskSnapshotList {
+	GinkgoHelper()
+	vdSnapshots := virtv2.VirtualDiskSnapshotList{}
+	err := GetObjects(kc.ResourceVDSnapshot, &vdSnapshots, kc.GetOptions{
+		ExcludedLabels: []string{"hasNoConsumer"},
+		Namespace:      namespace,
+		Labels:         labels,
+	})
+	Expect(err).NotTo(HaveOccurred(), "cannot get `vdSnapshots`\nstderr: %s", err)
+	return vdSnapshots
+}
+
 func CheckFileSystemFrozen(vmName, vmNamespace string) (bool, error) {
 	vmObj := virtv2.VirtualMachine{}
 	err := GetObject(kc.ResourceVM, vmName, &vmObj, kc.GetOptions{Namespace: vmNamespace})

tests/e2e/vm_connectivity_test.go

@@ -308,8 +308,12 @@ func CheckCiliumAgents(kubectl kc.Kubectl, namespace string, vms ...string) {
 	GinkgoHelper()
 	for _, vm := range vms {
 		By(fmt.Sprintf("Cilium agent should be OK's for VM: %s", vm))
-		err := network.CheckCilliumAgents(context.Background(), kubectl, vm, namespace)
-		Expect(err).NotTo(HaveOccurred())
+		Eventually(func() error {
+			return network.CheckCilliumAgents(context.Background(), kubectl, vm, namespace)
+		}).
+			WithTimeout(Timeout).
+			WithPolling(Interval).
+			Should(Succeed())
 	}
 }