From 1e1f8819bfeba725749ce2e4ad22e2a3c4ab6037 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Mon, 5 Jul 2021 13:07:37 +0200 Subject: [PATCH 1/5] e2e: don't export KUBECONFIG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit modifies the e2e shell scripts so that the KUBECONFIG variable does not need to be exported. This will become important once we allow the e2e test suite to launch multiple clusters, e.g. to test multi-cluster connectivity. Signed-off-by: Lucas Servén Marín --- e2e/full-mesh.sh | 10 ++++---- e2e/lib.sh | 56 ++++++++++++++++++++++++++++---------------- e2e/location-mesh.sh | 6 ++--- 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/e2e/full-mesh.sh b/e2e/full-mesh.sh index a5f0d7cf..494115a4 100644 --- a/e2e/full-mesh.sh +++ b/e2e/full-mesh.sh @@ -4,9 +4,9 @@ setup_suite() { # shellcheck disable=SC2016 - $KUBECTL_BINARY patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=full"]}]}}}}' + _kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=full"]}]}}}}' block_until_ready_by_name kube-system kilo-userspace - $KUBECTL_BINARY wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m + _kubectl wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m } test_full_mesh_connectivity() { @@ -23,9 +23,9 @@ test_full_mesh_peer() { test_full_mesh_allowed_location_ips() { docker exec kind-cluster-kilo-control-plane ip address add 10.6.0.1/32 dev eth0 - $KUBECTL_BINARY annotate node kind-cluster-kilo-control-plane kilo.squat.ai/allowed-location-ips=10.6.0.1/32 + _kubectl annotate node kind-cluster-kilo-control-plane kilo.squat.ai/allowed-location-ips=10.6.0.1/32 assert_equals Unauthorized "$(retry 10 5 'IP is not yet routable' curl_pod -m 1 -s -k https://10.6.0.1:10250/healthz)" "should be able to make HTTP request to allowed location IP" - $KUBECTL_BINARY annotate node kind-cluster-kilo-control-plane kilo.squat.ai/allowed-location-ips- + _kubectl annotate node kind-cluster-kilo-control-plane kilo.squat.ai/allowed-location-ips- assert "retry 10 5 'IP is still routable' _not curl_pod -m 1 -s -k https://10.6.0.1:10250/healthz" "should not be able to make HTTP request to allowed location IP" docker exec kind-cluster-kilo-control-plane ip address delete 10.6.0.1/32 dev eth0 } @@ -39,5 +39,5 @@ test_reject_peer_empty_public_key() { } test_mesh_granularity_auto_detect() { - assert_equals "$($KGCTL_BINARY graph)" "$($KGCTL_BINARY graph --mesh-granularity full)" + assert_equals "$(_kgctl graph)" "$(_kgctl graph --mesh-granularity full)" } diff --git a/e2e/lib.sh b/e2e/lib.sh index 584ccc95..2de8af86 100755 --- a/e2e/lib.sh +++ b/e2e/lib.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -export KUBECONFIG="kind.yaml" +KUBECONFIG="kind.yaml" KIND_CLUSTER="kind-cluster-kilo" KIND_BINARY="${KIND_BINARY:-kind}" KUBECTL_BINARY="${KUBECTL_BINARY:-kubectl}" @@ -33,6 +33,21 @@ _not() { return 0 } +# _kubectl is a helper that calls kubectl with the --kubeconfig flag. +_kubectl() { + $KUBECTL_BINARY --kubeconfig="$KUBECONFIG" "$@" +} + +# _kgctl is a helper that calls kgctl with the --kubeconfig flag. +_kgctl() { + $KGCTL_BINARY --kubeconfig="$KUBECONFIG" "$@" +} + +# _kind is a helper that calls kind with the --kubeconfig flag. +_kind() { + $KIND_BINARY --kubeconfig="$KUBECONFIG" "$@" +} + create_interface() { docker run -d --name="$1" --rm --network=host --cap-add=NET_ADMIN --device=/dev/net/tun -v /var/run/wireguard:/var/run/wireguard -e WG_LOG_LEVEL=debug leonnicolas/boringtun --foreground --disable-drop-privileges true "$1" } @@ -42,7 +57,7 @@ delete_interface() { } create_peer() { - cat < /dev/null + _kind delete clusters $KIND_CLUSTER > /dev/null # Create the kind cluster. - $KIND_BINARY create cluster --name $KIND_CLUSTER --config ./kind-config.yaml + _kind create cluster --name $KIND_CLUSTER --config ./kind-config.yaml # Load the Kilo image into kind. docker tag "$KILO_IMAGE" squat/kilo:test + # This command does not accept the --kubeconfig flag, so call the command directly. $KIND_BINARY load docker-image squat/kilo:test --name $KIND_CLUSTER # Create the kubeconfig secret. - $KUBECTL_BINARY create secret generic kubeconfig --from-file=kubeconfig="$KUBECONFIG" -n kube-system + _kubectl create secret generic kubeconfig --from-file=kubeconfig="$KUBECONFIG" -n kube-system # Apply Kilo the the cluster. - $KUBECTL_BINARY apply -f ../manifests/crds.yaml - $KUBECTL_BINARY apply -f kilo-kind-userspace.yaml + _kubectl apply -f ../manifests/crds.yaml + _kubectl apply -f kilo-kind-userspace.yaml block_until_ready_by_name kube-system kilo-userspace - $KUBECTL_BINARY wait nodes --all --for=condition=Ready + _kubectl wait nodes --all --for=condition=Ready # Wait for CoreDNS. block_until_ready kube_system k8s-app=kube-dns # Ensure the curl helper is not scheduled on a control-plane node. - $KUBECTL_BINARY apply -f helper-curl.yaml + _kubectl apply -f helper-curl.yaml block_until_ready_by_name default curl - $KUBECTL_BINARY taint node $KIND_CLUSTER-control-plane node-role.kubernetes.io/master:NoSchedule- - $KUBECTL_BINARY apply -f https://raw.githubusercontent.com/heptoprint/adjacency/master/example.yaml + _kubectl taint node $KIND_CLUSTER-control-plane node-role.kubernetes.io/master:NoSchedule- + _kubectl apply -f https://raw.githubusercontent.com/heptoprint/adjacency/master/example.yaml block_until_ready_by_name adjacency adjacency } delete_cluster () { - $KIND_BINARY delete clusters $KIND_CLUSTER + _kind delete clusters $KIND_CLUSTER } curl_pod() { - $KUBECTL_BINARY get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" exec {} -- /bin/sh -c "curl $*" + _kubectl get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" --kubeconfig="$KUBECONFIG" exec {} -- /bin/sh -c "curl $*" } check_ping() { @@ -123,7 +139,7 @@ check_ping() { shift done - for ip in $($KUBECTL_BINARY get pods -l app.kubernetes.io/name=adjacency -o jsonpath='{.items[*].status.podIP}'); do + for ip in $(_kubectl get pods -l app.kubernetes.io/name=adjacency -o jsonpath='{.items[*].status.podIP}'); do if [ -n "$LOCAL" ]; then ping=$(curl -m 1 -s http://"$ip":8080/ping) else @@ -140,7 +156,7 @@ check_ping() { } check_adjacent() { - $KUBECTL_BINARY get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" exec {} -- /bin/sh -c 'curl -m 1 -s adjacency:8080/?format=fancy' + _kubectl get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" --kubeconfig="$KUBECONFIG" exec {} -- /bin/sh -c 'curl -m 1 -s adjacency:8080/?format=fancy' [ "$(curl_pod -m 1 -s adjacency:8080/?format=json | jq | grep -c true)" -eq "$1" ] } @@ -152,14 +168,14 @@ check_peer() { create_interface "$INTERFACE" docker run --rm --entrypoint=/usr/bin/wg "$KILO_IMAGE" genkey > "$INTERFACE" assert "create_peer $PEER $ALLOWED_IP 10 $(docker run --rm --entrypoint=/bin/sh -v "$PWD/$INTERFACE":/key "$KILO_IMAGE" -c 'cat /key | wg pubkey')" "should be able to create Peer" - assert "$KGCTL_BINARY showconf peer $PEER --mesh-granularity=$GRANULARITY > $PEER.ini" "should be able to get Peer configuration" + assert "_kgctl showconf peer $PEER --mesh-granularity=$GRANULARITY > $PEER.ini" "should be able to get Peer configuration" assert "docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/usr/bin/wg -v /var/run/wireguard:/var/run/wireguard -v $PWD/$PEER.ini:/peer.ini $KILO_IMAGE setconf $INTERFACE /peer.ini" "should be able to apply configuration from kgctl" docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/usr/bin/wg -v /var/run/wireguard:/var/run/wireguard -v "$PWD/$INTERFACE":/key "$KILO_IMAGE" set "$INTERFACE" private-key /key docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/sbin/ip "$KILO_IMAGE" address add "$ALLOWED_IP" dev "$INTERFACE" docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/sbin/ip "$KILO_IMAGE" link set "$INTERFACE" up docker run --rm --network=host --cap-add=NET_ADMIN --entrypoint=/sbin/ip "$KILO_IMAGE" route add 10.42/16 dev "$INTERFACE" assert "retry 10 5 '' check_ping --local" "should be able to ping Pods from host" - assert_equals "$($KGCTL_BINARY showconf peer "$PEER")" "$($KGCTL_BINARY showconf peer "$PEER" --mesh-granularity="$GRANULARITY")" "kgctl should be able to auto detect the mesh granularity" + assert_equals "$(_kgctl showconf peer "$PEER")" "$(_kgctl showconf peer "$PEER" --mesh-granularity="$GRANULARITY")" "kgctl should be able to auto detect the mesh granularity" rm "$INTERFACE" "$PEER".ini delete_peer "$PEER" delete_interface "$INTERFACE" diff --git a/e2e/location-mesh.sh b/e2e/location-mesh.sh index a0e8b56d..dc3723f6 100755 --- a/e2e/location-mesh.sh +++ b/e2e/location-mesh.sh @@ -4,9 +4,9 @@ setup_suite() { # shellcheck disable=SC2016 - $KUBECTL_BINARY patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=location"]}]}}}}' + _kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=location"]}]}}}}' block_until_ready_by_name kube-system kilo-userspace - $KUBECTL_BINARY wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m + _kubectl wait pod -l app.kubernetes.io/name=adjacency --for=condition=Ready --timeout 3m } test_location_mesh_connectivity() { @@ -22,5 +22,5 @@ test_location_mesh_peer() { } test_mesh_granularity_auto_detect() { - assert_equals "$($KGCTL_BINARY graph)" "$($KGCTL_BINARY graph --mesh-granularity location)" + assert_equals "$(_kgctl graph)" "$(_kgctl graph --mesh-granularity location)" } From c728870b49031a67ddfacab0c157abe06c39a0b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Mon, 5 Jul 2021 13:09:51 +0200 Subject: [PATCH 2/5] e2e: check_adjacent accept node number MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit modifies the logic of the check_adjacent helper so that rather than expecting the argument to be n^2+n it expects simply n. This makes it easier to update the caller when the number of nodes in the cluster changes. Signed-off-by: Lucas Servén Marín --- e2e/full-mesh.sh | 4 ++-- e2e/lib.sh | 2 +- e2e/location-mesh.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/e2e/full-mesh.sh b/e2e/full-mesh.sh index 494115a4..b306fa10 100644 --- a/e2e/full-mesh.sh +++ b/e2e/full-mesh.sh @@ -11,10 +11,10 @@ setup_suite() { test_full_mesh_connectivity() { assert "retry 30 5 '' check_ping" "should be able to ping all Pods" - assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 12" "adjacency should return the right number of successful pings" + assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings" echo "sleep for 30s (one reconciliation period) and try again..." sleep 30 - assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 12" "adjacency should return the right number of successful pings after reconciling" + assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings after reconciling" } test_full_mesh_peer() { diff --git a/e2e/lib.sh b/e2e/lib.sh index 2de8af86..d8baa1c2 100755 --- a/e2e/lib.sh +++ b/e2e/lib.sh @@ -157,7 +157,7 @@ check_ping() { check_adjacent() { _kubectl get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" --kubeconfig="$KUBECONFIG" exec {} -- /bin/sh -c 'curl -m 1 -s adjacency:8080/?format=fancy' - [ "$(curl_pod -m 1 -s adjacency:8080/?format=json | jq | grep -c true)" -eq "$1" ] + [ "$(curl_pod -m 1 -s adjacency:8080/?format=json | jq '.[].latencies[].ok' | grep -c true)" -eq $(($1*$1)) ] } check_peer() { diff --git a/e2e/location-mesh.sh b/e2e/location-mesh.sh index dc3723f6..bf42555c 100755 --- a/e2e/location-mesh.sh +++ b/e2e/location-mesh.sh @@ -11,10 +11,10 @@ setup_suite() { test_location_mesh_connectivity() { assert "retry 30 5 '' check_ping" "should be able to ping all Pods" - assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 12" "adjacency should return the right number of successful pings" + assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings" echo "sleep for 30s (one reconciliation period) and try again..." sleep 30 - assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 12" "adjacency should return the right number of successful pings after reconciling" + assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings after reconciling" } test_location_mesh_peer() { From f81d19e6921237024f15f12c208aa48a6f2127c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Mon, 5 Jul 2021 13:14:23 +0200 Subject: [PATCH 3/5] e2e: allow parameterizing kind config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit allows the kind cluster configuration to be parameterized at call time. This enables the test suite to build multiple clusters with different configurations, e.g. different CIDRs, different numbers of nodes, etc. Signed-off-by: Lucas Servén Marín --- e2e/kind-config.yaml | 9 ++++----- e2e/lib.sh | 20 +++++++++++++++++++- e2e/setup.sh | 2 +- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/e2e/kind-config.yaml b/e2e/kind-config.yaml index 7e52b8c4..ba654731 100644 --- a/e2e/kind-config.yaml +++ b/e2e/kind-config.yaml @@ -1,11 +1,10 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: -- role: control-plane -- role: worker -- role: worker +- role: control-plane$WORKERS networking: disableDefaultCNI: true # disable kindnet - podSubnet: 10.42.0.0/16 apiServerAddress: 172.18.0.1 - apiServerPort: 6443 + apiServerPort: $API_SERVER_PORT + podSubnet: $POD_SUBNET + serviceSubnet: $SERVICE_SUBNET diff --git a/e2e/lib.sh b/e2e/lib.sh index d8baa1c2..64ef9c5d 100755 --- a/e2e/lib.sh +++ b/e2e/lib.sh @@ -48,6 +48,22 @@ _kind() { $KIND_BINARY --kubeconfig="$KUBECONFIG" "$@" } +# shellcheck disable=SC2120 +build_kind_config() { + local WORKER_COUNT="${1:-0}" + export API_SERVER_PORT="${2:-6443}" + export POD_SUBNET="${3:-10.42.0.0/16}" + export SERVICE_SUBNET="${4:-10.43.0.0/16}" + export WORKERS="" + local i=0 + while [ "$i" -lt "$WORKER_COUNT" ]; do + WORKERS="$(printf "%s\n- role: worker" "$WORKERS")" + ((i++)) + done + envsubst < ./kind-config.yaml + unset API_SERVER_PORT POD_SUBNET SERVICE_SUBNET WORKERS +} + create_interface() { docker run -d --name="$1" --rm --network=host --cap-add=NET_ADMIN --device=/dev/net/tun -v /var/run/wireguard:/var/run/wireguard -e WG_LOG_LEVEL=debug leonnicolas/boringtun --foreground --disable-drop-privileges true "$1" } @@ -96,9 +112,11 @@ block_until_ready() { # create_cluster launches a kind cluster and deploys Kilo, Adjacency, and a helper with curl. create_cluster() { + # shellcheck disable=SC2119 + local CONFIG="${1:-$(build_kind_config)}" _kind delete clusters $KIND_CLUSTER > /dev/null # Create the kind cluster. - _kind create cluster --name $KIND_CLUSTER --config ./kind-config.yaml + _kind create cluster --name $KIND_CLUSTER --config <(echo "$CONFIG") # Load the Kilo image into kind. docker tag "$KILO_IMAGE" squat/kilo:test # This command does not accept the --kubeconfig flag, so call the command directly. diff --git a/e2e/setup.sh b/e2e/setup.sh index 370774c2..9185c4a4 100644 --- a/e2e/setup.sh +++ b/e2e/setup.sh @@ -3,5 +3,5 @@ . lib.sh setup_suite() { - create_cluster + create_cluster "$(build_kind_config 2)" } From f251ddda98f33b368e153654dece36f340e335de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= Date: Mon, 5 Jul 2021 19:40:39 +0200 Subject: [PATCH 4/5] e2e: simplify check_adjacency helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit simplifies the check_adjacency helper to us the curl_pod helper rather than a re-written version of it. It also simplifies the curl_pod helper slightly to avoid the need for an additional shell. Signed-off-by: Lucas Servén Marín --- e2e/lib.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/lib.sh b/e2e/lib.sh index 64ef9c5d..7aafa364 100755 --- a/e2e/lib.sh +++ b/e2e/lib.sh @@ -143,7 +143,7 @@ delete_cluster () { } curl_pod() { - _kubectl get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" --kubeconfig="$KUBECONFIG" exec {} -- /bin/sh -c "curl $*" + _kubectl get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" --kubeconfig="$KUBECONFIG" exec {} -- /usr/bin/curl "$@" } check_ping() { @@ -174,7 +174,7 @@ check_ping() { } check_adjacent() { - _kubectl get pods -l app.kubernetes.io/name=curl -o name | xargs -I{} "$KUBECTL_BINARY" --kubeconfig="$KUBECONFIG" exec {} -- /bin/sh -c 'curl -m 1 -s adjacency:8080/?format=fancy' + curl_pod adjacency:8080/?format=fancy [ "$(curl_pod -m 1 -s adjacency:8080/?format=json | jq '.[].latencies[].ok' | grep -c true)" -eq $(($1*$1)) ] } From 86eea326dba51f15f7d53a8e00b4ca1b294609a6 Mon Sep 17 00:00:00 2001 From: leonnicolas Date: Mon, 5 Jul 2021 22:09:43 +0200 Subject: [PATCH 5/5] e2e/lib.sh: print to stderr in retry This way callers of retry can check against the returned value and not the logged strings. Signed-off-by: leonnicolas --- e2e/lib.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/lib.sh b/e2e/lib.sh index 7aafa364..04e6566c 100755 --- a/e2e/lib.sh +++ b/e2e/lib.sh @@ -16,9 +16,9 @@ retry() { if "$@"; then return 0 else - printf "%s(attempt %d/%d)\n" "$ERROR" "$c" "$COUNT" | color "$YELLOW" + printf "%s(attempt %d/%d)\n" "$ERROR" "$c" "$COUNT" | color "$YELLOW" 1>&2 if [ "$c" != "$COUNT" ]; then - printf "retrying in %d seconds...\n" "$SLEEP" | color "$YELLOW" + printf "retrying in %d seconds...\n" "$SLEEP" | color "$YELLOW" 1>&2 sleep "$SLEEP" fi fi