chore(e2e): fixes for more stability

- Ignore failed to watch, fail to elect, fail to create vmip errors in virtualization-controller log.
- Use Eventually to test IP in cilium agents.
- Fix handling kubectl errors in SaveResourcesForTest.

Signed-off-by: Ivan Mikheykin <[email protected]>

Author: diafour

.github/workflows/dev_module_build.yml

@@ -625,7 +625,7 @@ jobs:
         with:
           name: resources_from_failed_tests
           retention-days: 2
-          path: /tmp/e2e_failed__*
+          path: ${{ runner.temp }}/e2e_failed__*
           if-no-files-found: ignore
 
       - name: Cleanup E2E resources on cancel

tests/e2e/default_config.yaml

@@ -50,10 +50,16 @@ logFilter:
   - "does not have a pvc reference" # "err": "kvvm head-345e7b6a-testcases-image-hotplug/head-345e7b6a-vm-image-hotplug spec volume vi-head-345e7b6a-vi-alpine-http does not have a pvc reference"
   - "lastTransitionTime: Required value" # Err.
   - "virtualmachineipaddressleases.virtualization.deckhouse.io "
+  - "Failed to watch" # error if virtualization-controller restarts during tests. "msg": "Failed to watch", "err": "Get \"http://127.0.0.1:23915/apis/virtualization.deckhouse.io/v1alpha2/virtualmachinerestores?allowWatchBookmarks=true\u0026resourceVersion=709816257\u0026timeoutSeconds=310\u0026watch=true\": context canceled"
+  - "leader election lost"
 regexpLogFilter:
   - "failed to detach: .* not found" # "err" "failed to detach: virtualmachine.kubevirt.io \"head-497d17b-vm-automatic-with-hotplug\" not found",
   - "error patching .* not found" # "err" "error patching *** virtualimages.virtualization.deckhouse.io \"head-497d17b-vi-pvc-oref-vi-oref-vd\" not found",
   - "IP address .* is not among addresses assigned to 'default' network interface .*" # "msg": "IP address (10.66.10.61) is not among addresses assigned to 'default' network interface (10.66.10.60)"
+  - "Failed to update lock optimistically:.*leases.*leader-election-helper.*" # error during virtualization-controller lifecycle: attempt to reacquire leader election. "msg": "Failed to update lock optimistically: Put \"http://127.0.0.1:23915/apis/coordination.k8s.io/v1/namespaces/d8-virtualization/leases/d8-virt-operator-leader-election-helper?timeout=5s\": context deadline exceeded (Client.Timeout exceeded while awaiting headers), falling back to slow path"
+  - "Failed to update lock: .* leases.*leader-election-helper.*" # "msg": "ock: Operation cannot be fulfilled on leases.coordination.k8s.io \"d8-virt-operator-leader-election-helper\": the object has been modified; please apply your changes to the latest version and try again",
+  - "failed to create VirtualMachineIPAddress .* the specified IP address .* has already been allocated and has not been released" #     "err": "failed to create VirtualMachineIPAddress \"head-5d2c558-vm-restore-safe-tfv4w\": admission webhook \"vmip.virtualization-controller.validate.d8-virtualization\" denied the request: the VirtualMachineIPAddress cannot be created: the specified IP address 10.66.10.4 has already been allocated and has not been released"
+  - "error retrieving resource lock .*leader-election-helper" #  "msg": "error retrieving resource lock d8-virtualization/d8-virt-operator-leader-election-helper: context deadline exceeded",
 
 cleanupResources:
   - clustervirtualimages.virtualization.deckhouse.io

tests/e2e/network/cilium_agents.go

@@ -62,7 +62,7 @@ func CheckCilliumAgents(ctx context.Context, kubectl kc.Kubectl, vmName, vmNames
 			}
 
 			if !found {
-				return fmt.Errorf("failed: cilium agent %s for VM's node %s", pod.Name, nodeName)
+				return fmt.Errorf("failed: not found cilium agent %s for VM's node %s", pod.Name, nodeName)
 			}
 		} else {
 			// For pods on different nodes
@@ -72,7 +72,7 @@ func CheckCilliumAgents(ctx context.Context, kubectl kc.Kubectl, vmName, vmNames
 			}
 
 			if !found {
-				return fmt.Errorf("failed: cilium agent %s for node %s", pod.Name, pod.Spec.NodeName)
+				return fmt.Errorf("failed: not found cilium agent %s for node %s", pod.Name, pod.Spec.NodeName)
 			}
 		}
 	}

tests/e2e/util_test.go

@@ -734,6 +734,10 @@ func IsContainerRestarted(podName, containerName, namespace string, startedAt me
 	return false, fmt.Errorf("failed to compare the `startedAt` field before and after the tests ran: %s", podName)
 }
 
+// SaveTestResources dump some resources that may help in further diagnostic.
+//
+// NOTE: This method is called in AfterEach for failed specs only. Avoid to use Expect,
+// as it fails without reporting. Better use GinkgoWriter to report errors at this point.
 func SaveTestResources(labels map[string]string, additional string) {
 	replacer := strings.NewReplacer(
 		" ", "_",
@@ -743,16 +747,35 @@ func SaveTestResources(labels map[string]string, additional string) {
 		"(", "_",
 		")", "_",
 		"|", "_",
+		"`", "",
+		"'", "",
 	)
 	additional = replacer.Replace(strings.ToLower(additional))
 
-	str := fmt.Sprintf("/tmp/e2e_failed__%s__%s.yaml", labels["testcase"], additional)
+	tmpDir := os.Getenv("RUNNER_TEMP")
+	if tmpDir == "" {
+		tmpDir = "/tmp"
+	}
+	resFileName := fmt.Sprintf("%s/e2e_failed__%s__%s.yaml", tmpDir, labels["testcase"], additional)
+	errorFileName := fmt.Sprintf("%s/e2e_failed__%s__%s_error.txt", tmpDir, labels["testcase"], additional)
 
-	cmdr := kubectl.Get("virtualization,intvirt -A", kc.GetOptions{Output: "yaml", Labels: labels})
-	Expect(cmdr.Error()).NotTo(HaveOccurred(), "cmd: %s\nstderr: %s", cmdr.GetCmd(), cmdr.StdErr())
+	cmdr := kubectl.Get("virtualization,intvirt,po -A", kc.GetOptions{Output: "yaml", Labels: labels})
+	if cmdr.Error() != nil {
+		errReport := fmt.Sprintf("cmd: %s\nerror: %s\nstderr: %s\n", cmdr.GetCmd(), cmdr.Error(), cmdr.StdErr())
+		GinkgoWriter.Printf("Get resources error:\n%s\n", errReport)
+		err := os.WriteFile(errorFileName, []byte(errReport), 0o644)
+		if err != nil {
+			GinkgoWriter.Printf("Save error to file '%s' failed: %s\n", errorFileName, err)
+		}
+	}
 
-	err := os.WriteFile(str, cmdr.StdOutBytes(), 0o644)
-	Expect(err).NotTo(HaveOccurred(), "cmd: %s\nstderr: %s", cmdr.GetCmd(), cmdr.StdErr())
+	// Stdout may present even if error is occurred.
+	if len(cmdr.StdOutBytes()) > 0 {
+		err := os.WriteFile(resFileName, cmdr.StdOutBytes(), 0o644)
+		if err != nil {
+			GinkgoWriter.Printf("Save resources to file '%s' failed: %s\n", errorFileName, err)
+		}
+	}
 }
 
 type Watcher interface {

tests/e2e/vd_snapshots_test.go

@@ -319,7 +319,7 @@ var _ = Describe("VirtualDiskSnapshots", ginkgoutil.CommonE2ETestDecorators(), f
 						return nil
 					}
 					if frozen {
-						return fmt.Errorf("the filesystem of the virtual machine %s/%s is frozen", vm.Namespace, vm.Name)
+						return fmt.Errorf("the filesystem of the virtual machine %s/%s is still frozen", vm.Namespace, vm.Name)
 					}
 					return nil
 				}).WithTimeout(

tests/e2e/vm_connectivity_test.go

@@ -308,8 +308,12 @@ func CheckCiliumAgents(kubectl kc.Kubectl, namespace string, vms ...string) {
 	GinkgoHelper()
 	for _, vm := range vms {
 		By(fmt.Sprintf("Cilium agent should be OK's for VM: %s", vm))
-		err := network.CheckCilliumAgents(context.Background(), kubectl, vm, namespace)
-		Expect(err).NotTo(HaveOccurred())
+		Eventually(func() error {
+			return network.CheckCilliumAgents(context.Background(), kubectl, vm, namespace)
+		}).
+			WithTimeout(Timeout).
+			WithPolling(Interval).
+			Should(Succeed())
 	}
 }