Merge pull request #71 from cybozu-go/delete-host-veth-if-exists

Delete host-side veth if exists in CNI ADD.

.circleci/config.yml

@@ -103,6 +103,15 @@ jobs:
           runtime: docker
           suite: ./functions
           kubernetes-version: "1.15"
+  mtest-failures:
+    docker:
+      - image: google/cloud-sdk
+    steps:
+      - checkout
+      - run-mtest:
+          runtime: remote
+          suite: ./failures
+          kubernetes-version: "1.15"
   mtest-k8s-v1_14:
     docker:
       - image: google/cloud-sdk
@@ -132,6 +141,7 @@ workflows:
             - build
       - mtest-containerd
       - mtest-docker
+      - mtest-failures
       - mtest-k8s-v1_14
       - mtest-k8s-v1_13
   release:

cni/add.go

@@ -96,12 +96,22 @@ func setupVethPair(contVethName, namespace, podname string, mtu int, hostNS ns.N
 	if err = netlink.LinkSetUp(contVeth); err != nil {
 		return net.Interface{}, net.Interface{}, fmt.Errorf("failed to set %q up: %v", contVethName, err)
 	}
+	if injectFailures {
+		panicForFirstRun("AfterMakingVethPairInContainerNS")
+	}
 
 	hostVeth, err := netlink.LinkByName(hostVethName)
 	if err != nil {
 		return net.Interface{}, net.Interface{}, fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
 	}
 
+	err = hostNS.Do(func(_ ns.NetNS) error {
+		return deleteVethIfExists(hostVethName)
+	})
+	if err != nil {
+		return net.Interface{}, net.Interface{}, fmt.Errorf("failed to delete host veth %q: %v", hostVethName, err)
+	}
+
 	if err = netlink.LinkSetNsFd(hostVeth, int(hostNS.Fd())); err != nil {
 		return net.Interface{}, net.Interface{}, fmt.Errorf("failed to move veth to host netns: %v", err)
 	}
@@ -143,8 +153,9 @@ func generateHostVethName(prefix, namespace, podname string) string {
 func makeVeth(name, namespace, podname string, mtu int) (peerName string, veth netlink.Link, err error) {
 	peerName = generateHostVethName("veth", namespace, podname)
 
-	err = deletePeerIfExists(peerName)
+	err = deleteVethIfExists(peerName)
 	if err != nil {
+		err = fmt.Errorf("failed to delete peer veth %q: %v", peerName, err)
 		return
 	}
 
@@ -179,14 +190,16 @@ func makeVethPair(name, peer string, mtu int) (netlink.Link, error) {
 	return veth2, nil
 }
 
-func deletePeerIfExists(name string) error {
-	if oldHostVeth, err := netlink.LinkByName(name); err == nil {
-		err := netlink.LinkDel(oldHostVeth)
-		if err != nil {
-			return err
-		}
+func deleteVethIfExists(name string) error {
+	oldVeth, err := netlink.LinkByName(name)
+	switch err.(type) {
+	case nil:
+		return netlink.LinkDel(oldVeth)
+	case netlink.LinkNotFoundError:
+		return nil
+	default:
+		return err
 	}
-	return nil
 }
 
 // addRouteInHost does:
@@ -288,6 +301,9 @@ func Add(args *skel.CmdArgs) error {
 	if err != nil {
 		return err
 	}
+	if injectFailures {
+		panicForFirstRun("AfterMakingVethPair")
+	}
 
 	ip, err := getIPFromCoild(coildURL, podNS, podName, args.ContainerID)
 	if err != nil {
@@ -298,6 +314,9 @@ func Add(args *skel.CmdArgs) error {
 			returnIPToCoild(coildURL, podNS, podName, args.ContainerID)
 		}
 	}()
+	if injectFailures {
+		panicForFirstRun("AfterGettingIP")
+	}
 
 	err = addRouteInHost(ip, hostInterface.Name)
 	if err != nil {
@@ -318,6 +337,9 @@ func Add(args *skel.CmdArgs) error {
 	if err != nil {
 		return err
 	}
+	if injectFailures {
+		panicForFirstRun("AfterConfiguringInterfaces")
+	}
 
 	success = true
 	return types.PrintResult(result, conf.CNIVersion)

cni/del.go

@@ -31,6 +31,9 @@ func Del(args *skel.CmdArgs) error {
 	if err != nil {
 		return err
 	}
+	if injectFailures {
+		panicForFirstRun("AfterReturingIP")
+	}
 
 	// Kubernetes sends multiple DEL for dead containers.
 	// See: https://github.com/kubernetes/kubernetes/issues/44100
@@ -41,6 +44,9 @@ func Del(args *skel.CmdArgs) error {
 	return ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error {
 		err := ip.DelLinkByName(args.IfName)
 		if err == nil || err == ip.ErrLinkNotFound {
+			if injectFailures {
+				panicForFirstRun("AfterDeletingVethPair")
+			}
 			return nil
 		}
 		return err

cni/failures_false.go

@@ -0,0 +1,7 @@
+// +build !failures
+
+package cni
+
+const injectFailures = false
+
+func panicForFirstRun(name string) {}

cni/failures_true.go

@@ -0,0 +1,31 @@
+// +build failures
+
+package cni
+
+import (
+	"os"
+	"path/filepath"
+)
+
+const injectFailures = true
+
+// panicForFirstRun panics at least for the first run.
+// This "first" is judged against the lifetime of "/tmp" files.
+// This does not consider race conditions.
+func panicForFirstRun(name string) {
+	fileName := filepath.Join("/tmp", "coil_failures_"+name)
+
+	_, err := os.Stat(fileName)
+	if err == nil {
+		return
+	}
+	if !os.IsNotExist(err) {
+		panic(err)
+	}
+
+	_, err = os.Create(fileName)
+	if err != nil {
+		panic(err)
+	}
+	panic("injected failure: " + name)
+}

docs/mtest.md

@@ -12,6 +12,10 @@ There are following types of test suites.
 
     This suite tests coil controller, coil installer, `coilctl` command and kubernetes workloads deployments.
 
+2. failures
+
+    This suite tests kubernetes workloads deployments under failure injection.
+
 Each test suite has an entry point of test as `<suite>/suite_test.go`.
 
 Synopsis
@@ -45,7 +49,7 @@ Options
 ### `SUITE`
 
 You can choose the type of test suite by specifying `SUITE` make variable.
-The value can be `functions` (default).
+The value can be `functions` (default) or `failures`.
 
 `make test` accepts this variable.
 

mtest/Makefile

@@ -29,6 +29,9 @@ ifeq ($(findstring /,$(SUITE)),)
 else
   SUITE_PACKAGE = $(SUITE)
 endif
+ifeq ($(SUITE_PACKAGE),./failures)
+  HYPERCOIL_TAGS := $(HYPERCOIL_TAGS) -tags failures
+endif
 
 # If you want to change the Kubernetes version for mtest, specify this variable from command line.
 # e.g. $ make KUBERNETES_VERSION=1.14 placemat
@@ -105,15 +108,15 @@ all:
 $(OUTPUT)/coil.img:
 	rm -rf tmpbin
 	mkdir tmpbin
-	cd ..; GOBIN=$(shell pwd)/tmpbin CGO_ENABLED=0 go install -mod=vendor ./pkg/hypercoil
+	cd ..; GOBIN=$(shell pwd)/tmpbin CGO_ENABLED=0 go install -mod=vendor $(HYPERCOIL_TAGS) ./pkg/hypercoil
 	ln -s hypercoil tmpbin/coil
 	ln -s hypercoil tmpbin/coild
 	ln -s hypercoil tmpbin/coil-controller
 	ln -s hypercoil tmpbin/coilctl
 	ln -s hypercoil tmpbin/coil-installer
 	sudo docker build --no-cache --rm=false -f Dockerfile -t quay.io/cybozu/coil:dev tmpbin
 	mkdir -p $(OUTPUT)
-	sudo docker save -o $@ quay.io/cybozu/coil:dev
+	sudo docker save quay.io/cybozu/coil:dev > $@
 
 $(OUTPUT)/coilctl: $(OUTPUT)/coil.img
 	cp tmpbin/coilctl $@

mtest/failures.go

@@ -0,0 +1,89 @@
+package mtest
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+)
+
+// TestPodStartup tests pod startup, especially under failure injection.
+func TestPodStartup() {
+	BeforeEach(func() {
+		initializeCoil()
+		coilctlSafe("pool", "create", "default", "10.0.1.0/24", "2")
+	})
+	AfterEach(cleanCoil)
+
+	It("should confirm successful creation of CKE-managed pods", func() {
+		By("waiting for cluster-dns pods to become ready")
+		Eventually(func() error {
+			stdout, stderr, err := kubectl("get", "-n", "kube-system", "deployment", "cluster-dns", "-o", "json")
+			if err != nil {
+				return fmt.Errorf("err=%v, stdout=%s, stderr=%s", err, stdout, stderr)
+			}
+
+			deployment := new(appsv1.Deployment)
+			err = json.Unmarshal(stdout, deployment)
+			if err != nil {
+				return fmt.Errorf("err=%v, stdout=%s", err, stdout)
+			}
+
+			if deployment.Status.ReadyReplicas != 2 {
+				return errors.New("ReadyReplicas != 2")
+			}
+			return nil
+		}).Should(Succeed())
+	})
+
+	It("should starts up pod on half-cleaned node", func() {
+		By("making uncleaned veth artificially")
+		h := sha1.New()
+		h.Write([]byte("mtest.foo"))
+		vethName := "veth" + hex.EncodeToString(h.Sum(nil))[:11]
+		execSafeAt(node1, "sudo", "ip", "netns", "add", "foo")
+		execSafeAt(node1, "sudo", "ip", "netns", "exec", "foo", "ip", "link", "add", "eth0", "type", "veth", "peer", "name", vethName)
+		execSafeAt(node1, "sudo", "ip", "netns", "exec", "foo", "ip", "link", "set", vethName, "netns", "1")
+
+		By("creating pod with specific name on specific node")
+		podYAML := `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: foo
+  namespace: mtest
+spec:
+  nodeName: %s
+  containers:
+  - name: nginx
+    image: nginx
+`
+		stdout, stderr, err := kubectlWithInput([]byte(fmt.Sprintf(podYAML, node1)), "apply", "-f", "-")
+		Expect(err).NotTo(HaveOccurred(), "stdout: %s, stderr: %s", stdout, stderr)
+
+		By("waiting for pod to become ready")
+		Eventually(func() error {
+			stdout, stderr, err := kubectl("get", "pod", "foo", "-o", "json")
+			if err != nil {
+				return fmt.Errorf("err=%v, stdout=%s, stderr=%s", err, stdout, stderr)
+			}
+
+			pod := new(corev1.Pod)
+			err = json.Unmarshal(stdout, pod)
+			if err != nil {
+				return fmt.Errorf("err=%v, stdout=%s", err, stdout)
+			}
+
+			if pod.Status.Phase != corev1.PodRunning {
+				return errors.New("Phase != Running")
+			}
+			return nil
+		}).Should(Succeed())
+	})
+}

mtest/failures/suite_test.go

@@ -0,0 +1,28 @@
+package mtest
+
+import (
+	"os"
+	"testing"
+
+	"github.com/cybozu-go/coil/mtest"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestMtest(t *testing.T) {
+	if os.Getenv("SSH_PRIVKEY") == "" {
+		t.Skip("no SSH_PRIVKEY envvar")
+	}
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Multi-host test for coil")
+}
+
+var _ = BeforeSuite(func() {
+	mtest.RunBeforeSuite()
+})
+
+// This must be the only top-level test container.
+// Other tests and test containers must be listed in this.
+var _ = Describe("Test CKE functions", func() {
+	mtest.FailuresSuite()
+})

mtest/suites.go

@@ -9,3 +9,8 @@ var FunctionsSuite = func() {
 	Context("pod", TestPod)
 	Context("pool", TestPool)
 }
+
+// FailuresSuite is a test suite that runs test cases with failure injection
+var FailuresSuite = func() {
+	Context("pod startup", TestPodStartup)
+}