diff --git a/.gitignore b/.gitignore index 1e8d72ce..adc79293 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ .push* bin/ tmp/ -e2e/kind.yaml +e2e/kind.yaml* diff --git a/Makefile b/Makefile index 907abf1b..f7d02f2c 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,7 @@ EMBEDMD_BINARY := bin/embedmd KIND_BINARY := $(shell pwd)/bin/kind KUBECTL_BINARY := $(shell pwd)/bin/kubectl BASH_UNIT := $(shell pwd)/bin/bash_unit +BASH_UNIT_FLAGS := BUILD_IMAGE ?= golang:1.16.5-alpine BASE_IMAGE ?= alpine:3.13 @@ -208,7 +209,7 @@ $(BASH_UNIT): chmod +x $@ e2e: container $(KIND_BINARY) $(KUBECTL_BINARY) $(BASH_UNIT) bin/$(OS)/$(ARCH)/kgctl - KILO_IMAGE=$(IMAGE):$(ARCH)-$(VERSION) KIND_BINARY=$(KIND_BINARY) KUBECTL_BINARY=$(KUBECTL_BINARY) KGCTL_BINARY=$(shell pwd)/bin/$(OS)/$(ARCH)/kgctl $(BASH_UNIT) ./e2e/setup.sh ./e2e/full-mesh.sh ./e2e/location-mesh.sh ./e2e/teardown.sh + KILO_IMAGE=$(IMAGE):$(ARCH)-$(VERSION) KIND_BINARY=$(KIND_BINARY) KUBECTL_BINARY=$(KUBECTL_BINARY) KGCTL_BINARY=$(shell pwd)/bin/$(OS)/$(ARCH)/kgctl $(BASH_UNIT) $(BASH_UNIT_FLAGS) ./e2e/setup.sh ./e2e/full-mesh.sh ./e2e/location-mesh.sh ./e2e/multi-cluster.sh ./e2e/teardown.sh header: .header @HEADER=$$(cat .header); \ diff --git a/e2e/multi-cluster.sh b/e2e/multi-cluster.sh new file mode 100755 index 00000000..6eb58ce9 --- /dev/null +++ b/e2e/multi-cluster.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1091 +. lib.sh + +# shellcheck disable=SC2153 +KUBECONFIG2="$KUBECONFIG"2 +# shellcheck disable=SC2153 +KIND_CLUSTER2="$KIND_CLUSTER"2 + +setup_suite() { + KUBECONFIG=$KUBECONFIG2 KIND_CLUSTER=$KIND_CLUSTER2 create_cluster "$(build_kind_config 1 6444 10.44.0.0/16 10.45.0.0/16)" + # shellcheck disable=SC2016 + KUBECONFIG=$KUBECONFIG2 _kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=full","--subnet=10.6.0.0/16"]}]}}}}' + KUBECONFIG=$KUBECONFIG2 block_until_ready_by_name kube-system kilo-userspace + # Register the nodes in cluster1 as peers of cluster2. + for n in $(_kubectl get no -o name | cut -d'/' -f2); do + # Specify the service CIDR as an extra IP range that should be routable. + $KGCTL_BINARY --kubeconfig "$KUBECONFIG" showconf node "$n" --as-peer -o yaml --allowed-ips 10.43.0.0/16 | $KUBECTL_BINARY --kubeconfig "$KUBECONFIG2" apply -f - + done + # Register the nodes in cluster2 as peers of cluster1. + for n in $(KUBECONFIG=$KUBECONFIG2 _kubectl get no -o name | cut -d'/' -f2); do + # Specify the service CIDR as an extra IP range that should be routable. + $KGCTL_BINARY --kubeconfig "$KUBECONFIG2" showconf node "$n" --as-peer -o yaml --allowed-ips 10.45.0.0/16 | $KUBECTL_BINARY --kubeconfig "$KUBECONFIG" apply -f - + done +} + +test_multi_cluster_pod_connectivity() { + for ip in $(KUBECONFIG=$KUBECONFIG2 _kubectl get pods -l app.kubernetes.io/name=adjacency -o jsonpath='{.items[*].status.podIP}'); do + assert_equals pong "$(retry 10 5 "$ip is not yet routable" curl_pod -m 1 -s http://"$ip":8080/ping)" "should be able to make HTTP request from cluster 1 to Pod in cluster 2" + done + for ip in $(_kubectl get pods -l app.kubernetes.io/name=adjacency -o jsonpath='{.items[*].status.podIP}'); do + assert_equals pong "$(KUBECONFIG="$KUBECONFIG2" retry 10 5 "$ip is not yet routable" curl_pod -m 1 -s http://"$ip":8080/ping)" "should be able to make HTTP request from cluster 2 to Pod in cluster 1" + done +} + +test_multi_cluster_service_connectivity() { + # Mirror the Kubernetes API service from cluster1 into cluster2. + cat <