From 70a52d7e01e29ec4763876a1bd9601bae9f11625 Mon Sep 17 00:00:00 2001 From: Jonathan Tong Date: Thu, 3 Oct 2024 15:19:35 -0400 Subject: [PATCH 1/6] Refactor load test template to custom K8s builds template --- .../cluster-template-custom-builds-load.yaml} | 453 +++++++++++++++++- .../custom-builds-load}/kustomization.yaml | 2 +- .../patches/cluster-label-azuredisk.yaml | 0 .../patches/cluster-label-storageclass.yaml | 0 .../patches/kcp-scheduler.yaml | 0 .../storageclass-resource-set.yaml | 0 6 files changed, 448 insertions(+), 7 deletions(-) rename templates/test/{ci/cluster-template-prow-load.yaml => dev/cluster-template-custom-builds-load.yaml} (66%) rename templates/test/{ci/prow-load => dev/custom-builds-load}/kustomization.yaml (96%) rename templates/test/{ci/prow-load => dev/custom-builds-load}/patches/cluster-label-azuredisk.yaml (100%) rename templates/test/{ci/prow-load => dev/custom-builds-load}/patches/cluster-label-storageclass.yaml (100%) rename templates/test/{ci/prow-load => dev/custom-builds-load}/patches/kcp-scheduler.yaml (100%) rename templates/test/{ci/prow-load => dev/custom-builds-load}/storageclass-resource-set.yaml (100%) diff --git a/templates/test/ci/cluster-template-prow-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml similarity index 66% rename from templates/test/ci/cluster-template-prow-load.yaml rename to templates/test/dev/cluster-template-custom-builds-load.yaml index 328e38b83e1..b2b0b302908 100644 --- a/templates/test/ci/cluster-template-prow-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load.yaml @@ -8,6 +8,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + metrics-server: enabled storageclass: "true" name: ${CLUSTER_NAME} namespace: default @@ -54,6 +55,8 @@ spec: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane metadata: + annotations: + controlplane.cluster.x-k8s.io/skip-kube-proxy: "true" name: ${CLUSTER_NAME}-control-plane namespace: default spec: @@ -69,12 +72,14 @@ spec: allocate-node-cidrs: "false" cloud-provider: external cluster-name: ${CLUSTER_NAME} + feature-gates: HPAContainerMetrics=true v: "4" etcd: local: dataDir: /var/lib/etcddisk/etcd extraArgs: quota-backend-bytes: "8589934592" + kubernetesVersion: ci/${CI_VERSION} scheduler: extraArgs: authorization-always-allow-paths: /healthz,/readyz,/livez,/metrics @@ -104,21 +109,99 @@ spec: owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + systemctl stop kubelet + declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + az login --identity + for BINARY in "$${BINARIES[@]}"; do + echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login + done + systemctl restart kubelet + + # prepull images from gcr.io/k8s-staging-ci-images and retag it to + # registry.k8s.io so kubeadm can fetch correct images no matter what + declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for IMAGE in "$${IMAGES[@]}"; do + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"${CI_VERSION//+/_}" + done + + echo "kubeadm version: $(kubeadm version -o=short)" + echo "kubectl version: $(kubectl version --client=true)" + echo "kubelet version: $(kubelet --version)" + owner: root:root + path: /tmp/replace-k8s-binaries.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz + tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq + rm /tmp/yq_linux_amd64.tar.gz + + export KUBECONFIG=/etc/kubernetes/admin.conf + kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${KUBE_IMAGE_TAG}" + systemctl stop kubelet + yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml + yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml + yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml + systemctl restart kubelet + owner: root:root + path: /tmp/replace-k8s-components.sh + permissions: "0744" initConfiguration: nodeRegistration: kubeletExtraArgs: cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] + postKubeadmCommands: + - bash -c /tmp/replace-k8s-components.sh + preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/replace-k8s-binaries.sh + verbosity: 5 machineTemplate: infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -140,12 +223,18 @@ spec: lun: 0 nameSuffix: etcddisk identity: UserAssigned + image: + marketplace: + offer: capi + publisher: cncf-upstream + sku: ubuntu-2204-gen1 + version: latest osDisk: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -183,12 +272,18 @@ spec: template: spec: identity: UserAssigned + image: + marketplace: + offer: capi + publisher: cncf-upstream + sku: ubuntu-2204-gen1 + version: latest osDisk: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} vmExtensions: - name: CustomScript protectedSettings: @@ -216,12 +311,55 @@ spec: owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + systemctl stop kubelet + declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + az login --identity + for BINARY in "$${BINARIES[@]}"; do + echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login + done + systemctl restart kubelet + + echo "kubeadm version: $(kubeadm version -o=short)" + echo "kubectl version: $(kubectl version --client=true)" + echo "kubelet version: $(kubelet --version)" + owner: root:root + path: /tmp/replace-k8s-binaries.sh + permissions: "0744" joinConfiguration: nodeRegistration: kubeletExtraArgs: cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: [] + preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/replace-k8s-binaries.sh --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -261,6 +399,12 @@ spec: windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} spec: identity: UserAssigned + image: + marketplace: + offer: capi-windows + publisher: cncf-upstream + sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 + version: latest osDisk: diskSizeGB: 128 managedDisk: @@ -268,7 +412,7 @@ spec: osType: Windows sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 @@ -329,12 +473,68 @@ spec: sc.exe start sshd path: C:/collect-hns-crashes.ps1 permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + echo "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml + path: C:/oot-cred-provider.ps1 + permissions: "0744" + - content: | + Write-Host "Installing Azure CLI" + $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi + # Need to add manually AZ to PATH as it is not added without a reset + Write-Host "Adding AZ to path:" + $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" + Write-Host "Version Azure CLI:" + az version + path: C:/install-az-cli.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + Stop-Service kubelet -Force + + $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" + if($$KUBE_GIT_VERSION -ne "") + { + # Need to add manually AZ to PATH as it is not added without a reset + Write-Host "Adding AZ to path:" + $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" + # Install Binaries + $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") + $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" + echo "running az login via identity" + az login --identity + foreach ( $$binary in $$binaries ) + { + echo "downloading binary: $$ci_url/$$binary.exe" + az storage blob download --blob-url "$$ci_url/$$binary.exe" --file "c:/k/$$binary.exe" --auth-mode login + } + } + + # Tag it to the ci version. The image knows how to use the copy locally with the configmap + # that is applied at at this stage (windows-kubeproxy-ci.yaml) + ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess + ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" + + kubeadm.exe version -o=short + kubectl.exe version --client=true + kubelet.exe --version + kube-proxy.exe --version + path: C:/replace-pr-binaries.ps1 + permissions: "0744" joinConfiguration: nodeRegistration: criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: cloud-provider: external feature-gates: ${NODE_FEATURE_GATES:-""} + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml v: "2" windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS name: '{{ ds.meta_data["local_hostname"] }}' @@ -345,6 +545,9 @@ spec: - powershell C:/create-temp-folder.ps1 - powershell C:/replace-containerd.ps1 - powershell C:/collect-hns-crashes.ps1 + - powershell C:/oot-cred-provider.ps1 + - powershell C:/install-az-cli.ps1 + - powershell C:/replace-pr-binaries.ps1 users: - groups: Administrators name: capi @@ -526,6 +729,14 @@ spec: --- apiVersion: v1 data: + kube-proxy-patch: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: windows-kubeproxy-ci + namespace: kube-system + data: + KUBEPROXY_PATH: "c:/k/kube-proxy.exe" proxy: | apiVersion: apps/v1 kind: DaemonSet @@ -870,6 +1081,236 @@ metadata: --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet +metadata: + name: metrics-server-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + metrics-server: enabled + resources: + - kind: ConfigMap + name: metrics-server-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: v1 +data: + metrics-server: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader + rules: + - apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server + rules: + - apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get + - apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: v1 + kind: Service + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 + template: + metadata: + labels: + k8s-app: metrics-server + spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=4443 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + - --kubelet-insecure-tls + image: registry.k8s.io/metrics-server/metrics-server:v0.6.3 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 4443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - mountPath: /tmp + name: tmp-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + volumes: + - emptyDir: {} + name: tmp-dir + --- + apiVersion: apiregistration.k8s.io/v1 + kind: APIService + metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io + spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: metrics-server-${CLUSTER_NAME} + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet metadata: name: ${CLUSTER_NAME}-storageclass namespace: default diff --git a/templates/test/ci/prow-load/kustomization.yaml b/templates/test/dev/custom-builds-load/kustomization.yaml similarity index 96% rename from templates/test/ci/prow-load/kustomization.yaml rename to templates/test/dev/custom-builds-load/kustomization.yaml index 1857c1708c0..ae9cf29326f 100644 --- a/templates/test/ci/prow-load/kustomization.yaml +++ b/templates/test/dev/custom-builds-load/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: default resources: -- ../prow +- ../custom-builds - storageclass-resource-set.yaml patches: - path: patches/cluster-label-storageclass.yaml diff --git a/templates/test/ci/prow-load/patches/cluster-label-azuredisk.yaml b/templates/test/dev/custom-builds-load/patches/cluster-label-azuredisk.yaml similarity index 100% rename from templates/test/ci/prow-load/patches/cluster-label-azuredisk.yaml rename to templates/test/dev/custom-builds-load/patches/cluster-label-azuredisk.yaml diff --git a/templates/test/ci/prow-load/patches/cluster-label-storageclass.yaml b/templates/test/dev/custom-builds-load/patches/cluster-label-storageclass.yaml similarity index 100% rename from templates/test/ci/prow-load/patches/cluster-label-storageclass.yaml rename to templates/test/dev/custom-builds-load/patches/cluster-label-storageclass.yaml diff --git a/templates/test/ci/prow-load/patches/kcp-scheduler.yaml b/templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml similarity index 100% rename from templates/test/ci/prow-load/patches/kcp-scheduler.yaml rename to templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml diff --git a/templates/test/ci/prow-load/storageclass-resource-set.yaml b/templates/test/dev/custom-builds-load/storageclass-resource-set.yaml similarity index 100% rename from templates/test/ci/prow-load/storageclass-resource-set.yaml rename to templates/test/dev/custom-builds-load/storageclass-resource-set.yaml From 5fd17799e6898416d61468b0861833d6ea569be3 Mon Sep 17 00:00:00 2001 From: Jonathan Tong Date: Thu, 10 Oct 2024 15:27:35 -0400 Subject: [PATCH 2/6] Raise timeout to 1h and set create cluster attempts to 1 --- Makefile | 2 +- hack/create-dev-cluster.sh | 2 +- scripts/ci-entrypoint.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 13300eb3c62..1f83b7faf8f 100644 --- a/Makefile +++ b/Makefile @@ -364,7 +364,7 @@ create-workload-cluster: $(ENVSUBST) $(KUBECTL) ## Create a workload cluster. timeout --foreground 1800 bash -c "while ! $(KUBECTL) get secrets -n default | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done" # Get kubeconfig and store it locally. $(KUBECTL) get secret/$(CLUSTER_NAME)-kubeconfig -n default -o json | jq -r .data.value | base64 --decode > ./kubeconfig - $(KUBECTL) -n default wait --for=condition=Ready --timeout=10m cluster "$(CLUSTER_NAME)" + $(KUBECTL) -n default wait --for=condition=Ready --timeout=60m cluster "$(CLUSTER_NAME)" @echo 'run "$(KUBECTL) --kubeconfig=./kubeconfig ..." to work with the new target cluster' diff --git a/hack/create-dev-cluster.sh b/hack/create-dev-cluster.sh index 84ce1cef94d..fc08d519fef 100755 --- a/hack/create-dev-cluster.sh +++ b/hack/create-dev-cluster.sh @@ -28,7 +28,7 @@ make envsubst export REGISTRY="${REGISTRY:-registry.local/fake}" -export CLUSTER_CREATE_ATTEMPTS="${CLUSTER_CREATE_ATTEMPTS:-3}" +export CLUSTER_CREATE_ATTEMPTS="1" # Cluster settings. export CLUSTER_NAME="${CLUSTER_NAME:-capz-test}" diff --git a/scripts/ci-entrypoint.sh b/scripts/ci-entrypoint.sh index 0d0cfc5ab00..38be970c444 100755 --- a/scripts/ci-entrypoint.sh +++ b/scripts/ci-entrypoint.sh @@ -261,7 +261,7 @@ if [[ ! "${CLUSTER_TEMPLATE}" =~ "aks" ]]; then install_addons fi -"${KUBECTL}" --kubeconfig "${REPO_ROOT}/${KIND_CLUSTER_NAME}.kubeconfig" wait -A --for=condition=Ready --timeout=10m -l "cluster.x-k8s.io/cluster-name=${CLUSTER_NAME}" machinedeployments,machinepools +"${KUBECTL}" --kubeconfig "${REPO_ROOT}/${KIND_CLUSTER_NAME}.kubeconfig" wait -A --for=condition=Ready --timeout=60m -l "cluster.x-k8s.io/cluster-name=${CLUSTER_NAME}" machinedeployments,machinepools echo "Cluster ${CLUSTER_NAME} created and fully operational" From 3eae8f9e962dd87969a576b963c9b8675154aaab Mon Sep 17 00:00:00 2001 From: Jont828 Date: Mon, 21 Oct 2024 13:14:16 -0400 Subject: [PATCH 3/6] Bump KCP replicas to 6 --- templates/test/dev/cluster-template-custom-builds-load.yaml | 2 +- .../test/dev/custom-builds-load/patches/kcp-scheduler.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/test/dev/cluster-template-custom-builds-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml index b2b0b302908..4c1ee5fc212 100644 --- a/templates/test/dev/cluster-template-custom-builds-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load.yaml @@ -207,7 +207,7 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureMachineTemplate name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + replicas: 6 version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml b/templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml index 6e9d49617fe..d4c02a1f126 100644 --- a/templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml +++ b/templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml @@ -10,3 +10,4 @@ spec: extraArgs: authorization-always-allow-paths: /healthz,/readyz,/livez,/metrics bind-address: 0.0.0.0 + replicas: 6 From 86dd23601adc00b3aabaebd3690a23ab51cfc7c1 Mon Sep 17 00:00:00 2001 From: Jont828 Date: Mon, 21 Oct 2024 14:56:22 -0400 Subject: [PATCH 4/6] Bump to 7 --- templates/test/dev/cluster-template-custom-builds-load.yaml | 2 +- .../test/dev/custom-builds-load/patches/kcp-scheduler.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/test/dev/cluster-template-custom-builds-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml index 4c1ee5fc212..e7930f48014 100644 --- a/templates/test/dev/cluster-template-custom-builds-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load.yaml @@ -207,7 +207,7 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureMachineTemplate name: ${CLUSTER_NAME}-control-plane - replicas: 6 + replicas: 7 version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml b/templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml index d4c02a1f126..599fedbdce0 100644 --- a/templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml +++ b/templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml @@ -10,4 +10,4 @@ spec: extraArgs: authorization-always-allow-paths: /healthz,/readyz,/livez,/metrics bind-address: 0.0.0.0 - replicas: 6 + replicas: 7 From ceb90d2814a1a2d7d6aa22091e62fe77154f376b Mon Sep 17 00:00:00 2001 From: Jont828 Date: Mon, 21 Oct 2024 18:39:55 -0400 Subject: [PATCH 5/6] Remove replicas --- templates/test/dev/cluster-template-custom-builds-load.yaml | 2 +- .../test/dev/custom-builds-load/patches/kcp-scheduler.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/templates/test/dev/cluster-template-custom-builds-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml index e7930f48014..b2b0b302908 100644 --- a/templates/test/dev/cluster-template-custom-builds-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load.yaml @@ -207,7 +207,7 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureMachineTemplate name: ${CLUSTER_NAME}-control-plane - replicas: 7 + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml b/templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml index 599fedbdce0..6e9d49617fe 100644 --- a/templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml +++ b/templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml @@ -10,4 +10,3 @@ spec: extraArgs: authorization-always-allow-paths: /healthz,/readyz,/livez,/metrics bind-address: 0.0.0.0 - replicas: 7 From 38160771ff6c128f03a2d60bf8745acc9b11c2c3 Mon Sep 17 00:00:00 2001 From: Jont828 Date: Thu, 24 Oct 2024 12:21:38 -0400 Subject: [PATCH 6/6] Add ephemeral os disk config --- .../dev/cluster-template-custom-builds-load.yaml | 6 ++++++ .../test/dev/custom-builds-load/kustomization.yaml | 12 ++++++++++++ .../dev/custom-builds-load/patches/ephemeral.yaml | 7 +++++++ 3 files changed, 25 insertions(+) create mode 100644 templates/test/dev/custom-builds-load/patches/ephemeral.yaml diff --git a/templates/test/dev/cluster-template-custom-builds-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml index b2b0b302908..1f52650dcee 100644 --- a/templates/test/dev/cluster-template-custom-builds-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load.yaml @@ -230,6 +230,9 @@ spec: sku: ubuntu-2204-gen1 version: latest osDisk: + cachingType: ReadOnly + diffDiskSettings: + option: Local diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} @@ -279,6 +282,9 @@ spec: sku: ubuntu-2204-gen1 version: latest osDisk: + cachingType: ReadOnly + diffDiskSettings: + option: Local diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} diff --git a/templates/test/dev/custom-builds-load/kustomization.yaml b/templates/test/dev/custom-builds-load/kustomization.yaml index ae9cf29326f..ef7a1f09463 100644 --- a/templates/test/dev/custom-builds-load/kustomization.yaml +++ b/templates/test/dev/custom-builds-load/kustomization.yaml @@ -8,6 +8,18 @@ patches: - path: patches/cluster-label-storageclass.yaml - path: patches/cluster-label-azuredisk.yaml - path: patches/kcp-scheduler.yaml +- path: patches/ephemeral.yaml + target: + group: infrastructure.cluster.x-k8s.io + kind: AzureMachineTemplate + name: .*-md-0 + version: v1beta1 +- path: patches/ephemeral.yaml + target: + group: infrastructure.cluster.x-k8s.io + kind: AzureMachineTemplate + name: .*-control-plane + version: v1beta1 configMapGenerator: - files: - storageclass=../../../addons/storageclass-azure-disk.yaml diff --git a/templates/test/dev/custom-builds-load/patches/ephemeral.yaml b/templates/test/dev/custom-builds-load/patches/ephemeral.yaml new file mode 100644 index 00000000000..a19a2df6f7a --- /dev/null +++ b/templates/test/dev/custom-builds-load/patches/ephemeral.yaml @@ -0,0 +1,7 @@ +- op: replace + path: /spec/template/spec/osDisk/diffDiskSettings + value: + option: Local +- op: replace + path: /spec/template/spec/osDisk/cachingType + value: "ReadOnly"