Merge pull request #230 from inteon/improve_e2e_images

Preload images used in e2e tests

make/00_mod.mk

@@ -39,3 +39,12 @@ $(YQ) \
 	( .image.tag = "$(oci_manager_image_tag)" )' \
 	$1 --inplace
 endef
+
+mages_amd64 ?=
+images_arm64 ?=
+
+images_amd64 += docker.io/kong/httpbin:0.1.0@sha256:9d65a5b1955d2466762f53ea50eebae76be9dc7e277217cd8fb9a24b004154f4
+images_arm64 += docker.io/kong/httpbin:0.1.0@sha256:c546c8b06c542b615f053b577707cb72ddc875a0731d56d0ffaf840f767322ad
+
+images_amd64 += quay.io/curl/curl:8.5.0@sha256:e40a76dcfa9405678336774130411ca35beba85db426d5755b3cdd7b99d09a7a
+images_arm64 += quay.io/curl/curl:8.5.0@sha256:038b0290c9e4a371aed4f9d6993e3548fcfa32b96e9a170bfc73f5da4ec2354d

make/test-carotation.mk

@@ -15,13 +15,15 @@
 .PHONY: test-carotation
 ## carotation test
 ## @category Testing
-test-carotation: oci-load-manager | $(bin_dir)/scratch/istioctl-$(ISTIO_VERSION) $(NEEDS_KUBECTL) $(NEEDS_HELM) $(NEEDS_KIND) $(NEEDS_GOJQ)
+test-carotation: kind_cluster_name := "istio-csr-carotation"
+test-carotation: e2e-setup-cert-manager oci-load-manager | $(bin_dir)/scratch/istioctl-$(ISTIO_VERSION) $(NEEDS_KUBECTL) $(NEEDS_HELM) $(NEEDS_KIND) $(NEEDS_GOJQ)
 	$(eval oci_image_tar := $(bin_dir)/scratch/image/oci-layout-manager.$(oci_manager_image_tag).docker.tar)
 
 	ARTIFACTS=$(ARTIFACTS) \
 	ISTIO_CSR_IMAGE=$(oci_manager_image_name_development) \
 	ISTIO_CSR_IMAGE_TAR=$(oci_image_tar) \
 	ISTIO_CSR_IMAGE_TAG=$(oci_manager_image_tag) \
+	KIND_CLUSTER_NAME=$(kind_cluster_name) \
 	ISTIO_BIN=$(bin_dir)/scratch/istioctl-$(ISTIO_VERSION) \
 	KUBECTL_BIN=$(KUBECTL) \
 	HELM_BIN=$(HELM) \

test/carotation/cleanup-1.sh

@@ -21,10 +21,10 @@ set -o pipefail
 echo "======================================"
 echo ">> cleaning up resources"
 
-rm -f $TEST_DIR/ca.pem
+rm -f "$TEST_DIR/ca.pem"
 
 echo ">> exporting kind loads"
-$KIND_BIN export logs $ARTIFACTS --name istio-ca-rotation
+$KIND_BIN export logs "$ARTIFACTS" --name "$KIND_CLUSTER_NAME"
 
 echo ">> deleting cluster..."
-$KIND_BIN delete cluster --name istio-ca-rotation
+$KIND_BIN delete cluster --name "$KIND_CLUSTER_NAME"

test/carotation/env.sh

@@ -14,13 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-export TEST_DIR="${SCRIPT_DIR}"
-
 export ARTIFACTS="${ARTIFACTS:-./_bin/artifacts}"
 export ISTIO_CSR_IMAGE_TAR="${ISTIO_CSR_IMAGE_TAR:-./_bin/scratch/image/oci-layout-manager.v0.7.1.docker.tar}"
 export ISTIO_CSR_IMAGE="${ISTIO_CSR_IMAGE:-cert-manager.local/cert-manager-istio-csr}"
 export ISTIO_CSR_IMAGE_TAG="${ISTIO_CSR_IMAGE_TAG:-canary}"
+export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-istio-ca-rotation}"
 
 export ISTIO_BIN="${ISTIO_BIN:-./_bin/scratch/istioctl-1.17.2}"
 export KUBECTL_BIN="${KUBECTL_BIN:-./_bin/tools/kubectl}"

test/carotation/run.sh

@@ -18,25 +18,24 @@ set -o nounset
 set -o errexit
 set -o pipefail
 
+TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export TEST_DIR
 
-source ./test/carotation/env.sh
-
+source "$TEST_DIR/env.sh"
 
 # Ensure we always clean up after ourselves.
 cleanup() {
-  $TEST_DIR/cleanup-1.sh
+  "$TEST_DIR/cleanup-1.sh"
 }
 trap cleanup EXIT
 
 echo "======================================"
 echo ">> running CA rotation test"
 
-$TEST_DIR/setup-1.sh
-
-$TEST_DIR/setup-2.sh
+"$TEST_DIR/setup-1.sh"
 
-$TEST_DIR/setup-3.sh
+"$TEST_DIR/setup-2.sh"
 
-$TEST_DIR/test-1.sh
+"$TEST_DIR/test-1.sh"
 
-$TEST_DIR/test-2.sh
+"$TEST_DIR/test-2.sh"

test/carotation/setup-1.sh

@@ -19,17 +19,26 @@ set -o errexit
 set -o pipefail
 
 echo "======================================"
-echo ">> setting up CA rotation test cluster"
+echo ">> creating 2 roots of trust"
 
-echo ">> deleting any existing kind cluster..."
-$KIND_BIN delete cluster --name istio-ca-rotation
+echo ">> creating cert-manager issuers"
+$KUBECTL_BIN create namespace istio-system
+$KUBECTL_BIN apply -f "$TEST_DIR/issuers/."
 
-echo ">> creating kind cluster..."
-$KIND_BIN create cluster --name istio-ca-rotation
+echo ">> waiting for issuers to become ready"
+$KUBECTL_BIN get issuers -n istio-system
+$KUBECTL_BIN wait --timeout=180s -n istio-system --for=condition=ready issuer istio-root-1
+$KUBECTL_BIN wait --timeout=180s -n istio-system --for=condition=ready issuer istio-root-2
+$KUBECTL_BIN wait --timeout=180s -n istio-system --for=condition=ready issuer istio-int-1
+$KUBECTL_BIN wait --timeout=180s -n istio-system --for=condition=ready issuer istio-int-2
+$KUBECTL_BIN get issuers -n istio-system
 
-echo ">> loading docker image..."
-$KIND_BIN load image-archive $ISTIO_CSR_IMAGE_TAR --name istio-ca-rotation
+echo ">> extracting roots of trust"
+$KUBECTL_BIN get secret -n istio-system istio-root-1 -o jsonpath="{.data['ca\.crt']}" | base64 -d
+$KUBECTL_BIN get secret -n istio-system istio-root-1 -o jsonpath="{.data['ca\.crt']}" | base64 -d > "$TEST_DIR/ca.pem"
+$KUBECTL_BIN get secret -n istio-system istio-root-2 -o jsonpath="{.data['ca\.crt']}" | base64 -d
+$KUBECTL_BIN get secret -n istio-system istio-root-2 -o jsonpath="{.data['ca\.crt']}" | base64 -d >> "$TEST_DIR/ca.pem"
 
-echo ">> installing cert-manager"
-$HELM_BIN repo add jetstack https://charts.jetstack.io --force-update
-$HELM_BIN upgrade -i -n cert-manager cert-manager jetstack/cert-manager --set installCRDs=true --wait --create-namespace --set global.logLevel=2
+echo ">> creating roots of trust secret"
+cat "$TEST_DIR/ca.pem"
+$KUBECTL_BIN create secret generic istio-root-certs --from-file=ca.pem="$TEST_DIR/ca.pem" -n cert-manager

test/carotation/setup-2.sh

@@ -19,26 +19,27 @@ set -o errexit
 set -o pipefail
 
 echo "======================================"
-echo ">> creating 2 roots of trust"
+echo ">> installing istio-csr with roots of trust, using issuer from root-1"
 
-echo ">> creating cert-manager issuers"
-$KUBECTL_BIN create namespace istio-system
-$KUBECTL_BIN apply -f $TEST_DIR/issuers/.
+echo ">> installing cert-manager-istio-csr with first root"
+echo "$HELM_BIN upgrade -i cert-manager-istio-csr ./deploy/charts/istio-csr -n cert-manager --values $TEST_DIR/values/istio-csr-1.yaml --wait"
+$HELM_BIN upgrade -i cert-manager-istio-csr ./deploy/charts/istio-csr \
+  -n cert-manager \
+  --values "$TEST_DIR/values/istio-csr-1.yaml" \
+  --set image.repository="$ISTIO_CSR_IMAGE" \
+  --set image.tag="$ISTIO_CSR_IMAGE_TAG" \
+  --wait
 
-echo ">> waiting for issuers to become ready"
-$KUBECTL_BIN get issuers -n istio-system
-$KUBECTL_BIN wait --timeout=180s -n istio-system --for=condition=ready issuer istio-root-1
-$KUBECTL_BIN wait --timeout=180s -n istio-system --for=condition=ready issuer istio-root-2
-$KUBECTL_BIN wait --timeout=180s -n istio-system --for=condition=ready issuer istio-int-1
-$KUBECTL_BIN wait --timeout=180s -n istio-system --for=condition=ready issuer istio-int-2
-$KUBECTL_BIN get issuers -n istio-system
+echo ">> installing istio"
+$ISTIO_BIN install -y -f "$TEST_DIR/values/istio.yaml"
 
-echo ">> extracting roots of trust"
-$KUBECTL_BIN get secret -n istio-system istio-root-1 -o jsonpath="{.data['ca\.crt']}" | base64 -d
-$KUBECTL_BIN get secret -n istio-system istio-root-1 -o jsonpath="{.data['ca\.crt']}" | base64 -d > $TEST_DIR/ca.pem
-$KUBECTL_BIN get secret -n istio-system istio-root-2 -o jsonpath="{.data['ca\.crt']}" | base64 -d
-$KUBECTL_BIN get secret -n istio-system istio-root-2 -o jsonpath="{.data['ca\.crt']}" | base64 -d >> $TEST_DIR/ca.pem
-
-echo ">> creating roots of trust secret"
-cat $TEST_DIR/ca.pem
-$KUBECTL_BIN create secret generic istio-root-certs --from-file=ca.pem=$TEST_DIR/ca.pem -n cert-manager
+echo ">> enforcing mTLS everywhere"
+$KUBECTL_BIN apply -n istio-system -f - <<EOF
+apiVersion: "security.istio.io/v1beta1"
+kind: "PeerAuthentication"
+metadata:
+  name: "default"
+spec:
+  mtls:
+    mode: STRICT
+EOF

test/carotation/test-1.sh

@@ -22,10 +22,10 @@ echo "======================================"
 echo ">> installing workloads and testing connectivity"
 
 echo ">> installing workloads"
-$KUBECTL_BIN apply -f $TEST_DIR/workloads --wait --timeout=180s
+$KUBECTL_BIN apply -f "$TEST_DIR/workloads" --wait --timeout=180s
 $KUBECTL_BIN wait -n sandbox --for=condition=ready pod -l app=sleep --timeout=180s
 $KUBECTL_BIN wait -n sandbox --for=condition=ready pod -l app=httpbin --timeout=180s
 
 echo ">> testing mTLS connection between workloads"
 POD_NAME=$($KUBECTL_BIN get pod -n sandbox -l app=sleep -o jsonpath='{.items[0].metadata.name}')
-$KUBECTL_BIN exec $POD_NAME -c sleep -n sandbox -- curl -sS httpbin:8000/ip
+$KUBECTL_BIN exec "$POD_NAME" -c sleep -n sandbox -- curl -sS httpbin:8000/ip

test/carotation/test-2.sh

@@ -25,25 +25,25 @@ echo ">> reinstalling istio-csr with new issuer"
 $KUBECTL_BIN delete deploy -n cert-manager cert-manager-istio-csr --wait --timeout=180s
 $HELM_BIN upgrade -i cert-manager-istio-csr ./deploy/charts/istio-csr \
   -n cert-manager \
-  --values $TEST_DIR/values/istio-csr-2.yaml \
-  --set image.repository=$ISTIO_CSR_IMAGE \
-  --set image.tag=$ISTIO_CSR_IMAGE_TAG \
+  --values "$TEST_DIR/values/istio-csr-2.yaml" \
+  --set image.repository="$ISTIO_CSR_IMAGE" \
+  --set image.tag="$ISTIO_CSR_IMAGE_TAG" \
   --wait
 sleep 5s
 
 echo ">> rotating httpbin pod so it picks up new CA"
 POD_NAME=$($KUBECTL_BIN get pod -n sandbox -l app=httpbin -o jsonpath='{.items[0].metadata.name}')
 echo ">> current mTLS certificate"
-$ISTIO_BIN pc s $POD_NAME -n sandbox -o json | $JQ_BIN -r '.dynamicActiveSecrets[0].secret.tlsCertificate.certificateChain.inlineBytes' | base64 -d | openssl x509 --noout --text | grep Issuer:
+$ISTIO_BIN pc s "$POD_NAME" -n sandbox -o json | $JQ_BIN -r '.dynamicActiveSecrets[0].secret.tlsCertificate.certificateChain.inlineBytes' | base64 -d | openssl x509 --noout --text | grep Issuer:
 
-$KUBECTL_BIN delete po -n sandbox $POD_NAME --wait --timeout=180s
+$KUBECTL_BIN delete po -n sandbox "$POD_NAME" --wait --timeout=180s
 $KUBECTL_BIN wait -n sandbox --for=condition=ready pod -l app=httpbin --timeout=180s
 
 echo ">> new mTLS certificate"
 POD_NAME=$($KUBECTL_BIN get pod -n sandbox -l app=httpbin -o jsonpath='{.items[0].metadata.name}')
-$ISTIO_BIN pc s $POD_NAME -n sandbox -o json | $JQ_BIN -r '.dynamicActiveSecrets[0].secret.tlsCertificate.certificateChain.inlineBytes' | base64 -d | openssl x509 --noout --text | grep Issuer:
+$ISTIO_BIN pc s "$POD_NAME" -n sandbox -o json | $JQ_BIN -r '.dynamicActiveSecrets[0].secret.tlsCertificate.certificateChain.inlineBytes' | base64 -d | openssl x509 --noout --text | grep Issuer:
 
 
 echo ">> testing mTLS connection between workloads"
 POD_NAME=$($KUBECTL_BIN get pod -n sandbox -l app=sleep -o jsonpath='{.items[0].metadata.name}')
-$KUBECTL_BIN exec $POD_NAME -c sleep -n sandbox -- curl -sS httpbin:8000/ip
+$KUBECTL_BIN exec "$POD_NAME" -c sleep -n sandbox -- curl -sS httpbin:8000/ip

test/carotation/workloads/httpbin.yaml

@@ -63,8 +63,8 @@ spec:
     spec:
       serviceAccountName: httpbin
       containers:
-      - image: quay.io/joshvanl_jetstack/httpbin:latest
-        imagePullPolicy: IfNotPresent
+      - image: docker.io/kong/httpbin:0.1.0
+        imagePullPolicy: Never
         name: httpbin
         ports:
         - containerPort: 80

test/carotation/workloads/sleep.yaml

@@ -60,10 +60,10 @@ spec:
     spec:
       serviceAccountName: sleep
       containers:
-      - name: sleep
-        image: quay.io/curl/curl:latest
+      - image: quay.io/curl/curl:8.5.0
+        imagePullPolicy: Never
+        name: sleep
         command: ["/bin/sleep", "3650d"]
-        imagePullPolicy: IfNotPresent
         volumeMounts:
         - mountPath: /etc/sleep/tls
           name: secret-volume

test/e2e/manifests/httpbin.yaml

@@ -53,8 +53,8 @@ spec:
     spec:
       serviceAccountName: httpbin
       containers:
-      - image: kennethreitz/httpbin:latest
-        imagePullPolicy: IfNotPresent
+      - image: docker.io/kong/httpbin:0.1.0
+        imagePullPolicy: Never
         name: httpbin
         ports:
         - containerPort: 80

test/e2e/manifests/sleep.yaml

@@ -50,10 +50,10 @@ spec:
     spec:
       serviceAccountName: sleep
       containers:
-      - name: sleep
-        image: quay.io/curl/curl:latest
+      - image: quay.io/curl/curl:8.5.0
+        imagePullPolicy: Never
+        name: sleep
         command: ["/bin/sleep", "3650d"]
-        imagePullPolicy: IfNotPresent
         volumeMounts:
         - mountPath: /etc/sleep/tls
           name: secret-volume