diff --git a/templates/test/dev/cluster-template-custom-builds-dra.yaml b/templates/test/dev/cluster-template-custom-builds-dra.yaml index be845fa1be58..bce667352e10 100644 --- a/templates/test/dev/cluster-template-custom-builds-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-dra.yaml @@ -2,12 +2,14 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled metrics-server: enabled + storageclass: "true" name: ${CLUSTER_NAME} namespace: default spec: @@ -713,6 +715,20 @@ spec: name: metrics-server-${CLUSTER_NAME} strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-storageclass + namespace: default +spec: + clusterSelector: + matchLabels: + storageclass: "true" + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-storageclass + strategy: ApplyOnce +--- apiVersion: v1 data: metrics-server: | @@ -928,3 +944,57 @@ metadata: type: generated name: metrics-server-${CLUSTER_NAME} namespace: default +--- +apiVersion: v1 +data: + storageclass: | + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: default + annotations: + storageclass.beta.kubernetes.io/is-default-class: "true" + labels: + kubernetes.io/cluster-service: "true" + provisioner: kubernetes.io/azure-disk + parameters: + kind: Managed + storageaccounttype: Standard_LRS + cachingmode: ReadOnly + volumeBindingMode: WaitForFirstConsumer + --- + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: managed-premium + annotations: + labels: + kubernetes.io/cluster-service: "true" + provisioner: kubernetes.io/azure-disk + parameters: + kind: Managed + storageaccounttype: Premium_LRS + cachingmode: ReadOnly + volumeBindingMode: WaitForFirstConsumer + --- + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: managed-standard + annotations: + labels: + kubernetes.io/cluster-service: "true" + provisioner: kubernetes.io/azure-disk + parameters: + kind: Managed + storageaccounttype: Standard_LRS + cachingmode: ReadOnly + volumeBindingMode: WaitForFirstConsumer +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: storageclass-${CLUSTER_NAME} + namespace: default diff --git a/templates/test/dev/cluster-template-custom-builds-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml deleted file mode 100644 index b2b0b302908a..000000000000 --- a/templates/test/dev/cluster-template-custom-builds-load.yaml +++ /dev/null @@ -1,1378 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - azuredisk-csi: "true" - cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} - cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled - metrics-server: enabled - storageclass: "true" - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - buildProvenance: ${BUILD_PROVENANCE} - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - annotations: - controlplane.cluster.x-k8s.io/skip-kube-proxy: "true" - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-provider: external - feature-gates: ${K8S_FEATURE_GATES:-""} - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-provider: external - cluster-name: ${CLUSTER_NAME} - feature-gates: HPAContainerMetrics=true - v: "4" - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - kubernetesVersion: ci/${CI_VERSION} - scheduler: - extraArgs: - authorization-always-allow-paths: /healthz,/readyz,/livez,/metrics - bind-address: 0.0.0.0 - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" - - echo "Use OOT credential provider" - mkdir -p /var/lib/kubelet/credential-provider - curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" - chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider - curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" - chmod 644 /var/lib/kubelet/credential-provider-config.yaml - owner: root:root - path: /tmp/oot-cred-provider.sh - permissions: "0744" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - - systemctl stop kubelet - declare -a BINARIES=("kubeadm" "kubectl" "kubelet") - az login --identity - for BINARY in "$${BINARIES[@]}"; do - echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login - done - systemctl restart kubelet - - # prepull images from gcr.io/k8s-staging-ci-images and retag it to - # registry.k8s.io so kubeadm can fetch correct images no matter what - declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") - [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" - IMAGE_REGISTRY_PREFIX=registry.k8s.io - for IMAGE in "$${IMAGES[@]}"; do - $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"${CI_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"${CI_VERSION//+/_}" - done - - echo "kubeadm version: $(kubeadm version -o=short)" - echo "kubectl version: $(kubectl version --client=true)" - echo "kubelet version: $(kubelet --version)" - owner: root:root - path: /tmp/replace-k8s-binaries.sh - permissions: "0744" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - - curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz - tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq - rm /tmp/yq_linux_amd64.tar.gz - - export KUBECONFIG=/etc/kubernetes/admin.conf - kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${KUBE_IMAGE_TAG}" - systemctl stop kubelet - yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml - yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml - yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml - systemctl restart kubelet - owner: root:root - path: /tmp/replace-k8s-components.sh - permissions: "0744" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: - - bash -c /tmp/replace-k8s-components.sh - preKubeadmCommands: - - bash -c /tmp/oot-cred-provider.sh - - bash -c /tmp/replace-k8s-binaries.sh - verbosity: 5 - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - identity: UserAssigned - image: - marketplace: - offer: capi - publisher: cncf-upstream - sku: ubuntu-2204-gen1 - version: latest - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT:=2} - selector: {} - template: - metadata: - labels: - nodepool: pool1 - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - identity: UserAssigned - image: - marketplace: - offer: capi - publisher: cncf-upstream - sku: ubuntu-2204-gen1 - version: latest - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} - vmExtensions: - - name: CustomScript - protectedSettings: - commandToExecute: | - #!/bin/sh - echo "This script is a no-op used for extension testing purposes ..." - touch test_file - publisher: Microsoft.Azure.Extensions - version: "2.1" - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" - - echo "Use OOT credential provider" - mkdir -p /var/lib/kubelet/credential-provider - curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" - chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider - curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" - chmod 644 /var/lib/kubelet/credential-provider-config.yaml - owner: root:root - path: /tmp/oot-cred-provider.sh - permissions: "0744" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - - systemctl stop kubelet - declare -a BINARIES=("kubeadm" "kubectl" "kubelet") - az login --identity - for BINARY in "$${BINARIES[@]}"; do - echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login - done - systemctl restart kubelet - - echo "kubeadm version: $(kubeadm version -o=short)" - echo "kubectl version: $(kubectl version --client=true)" - echo "kubelet version: $(kubelet --version)" - owner: root:root - path: /tmp/replace-k8s-binaries.sh - permissions: "0744" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: - - bash -c /tmp/oot-cred-provider.sh - - bash -c /tmp/replace-k8s-binaries.sh ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - annotations: - runtime: containerd - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - spec: - identity: UserAssigned - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: |- - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - # /tmp is assumed created and required for upstream e2e tests to pass - New-Item -ItemType Directory -Force -Path C:\tmp\ - path: C:/create-temp-folder.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" - if($$CONTAINERD_URL -ne ""){ - # Kubelet service depends on contianerd service so make a best effort attempt to stop it - Stop-Service kubelet -Force -ErrorAction SilentlyContinue - Stop-Service containerd -Force - echo "downloading containerd: $$CONTAINERD_URL" - curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" - tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 - - Start-Service containerd - } - - containerd.exe --version - containerd-shim-runhcs-v1.exe --version - path: C:/replace-containerd.ps1 - permissions: "0744" - - content: | - mkdir -Force c:/localdumps - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f - # Enable sftp so we can copy crash dump files during log collection of stfp - $sshd_config = "$env:ProgramData\ssh\sshd_config" - if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } - Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" - sc.exe stop sshd - sc.exe start sshd - path: C:/collect-hns-crashes.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - echo "Use OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml - path: C:/oot-cred-provider.ps1 - permissions: "0744" - - content: | - Write-Host "Installing Azure CLI" - $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi - # Need to add manually AZ to PATH as it is not added without a reset - Write-Host "Adding AZ to path:" - $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" - Write-Host "Version Azure CLI:" - az version - path: C:/install-az-cli.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" - if($$KUBE_GIT_VERSION -ne "") - { - # Need to add manually AZ to PATH as it is not added without a reset - Write-Host "Adding AZ to path:" - $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" - # Install Binaries - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" - echo "running az login via identity" - az login --identity - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - az storage blob download --blob-url "$$ci_url/$$binary.exe" --file "c:/k/$$binary.exe" --auth-mode login - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-pr-binaries.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - feature-gates: ${NODE_FEATURE_GATES:-""} - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - v: "2" - windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell C:/create-temp-folder.ps1 - - powershell C:/replace-containerd.ps1 - - powershell C:/collect-hns-crashes.ps1 - - powershell C:/oot-cred-provider.ps1 - - powershell C:/install-az-cli.ps1 - - powershell C:/replace-pr-binaries.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineHealthCheck -metadata: - name: ${CLUSTER_NAME}-mhc-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - maxUnhealthy: 100% - selector: - matchLabels: - nodepool: pool1 - unhealthyConditions: - - status: "True" - timeout: 30s - type: E2ENodeUnhealthy ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} - tenantID: ${AZURE_TENANT_ID} - type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1alpha1 -kind: HelmChartProxy -metadata: - name: calico - namespace: default -spec: - chartName: tigera-operator - clusterSelector: - matchLabels: - cni: calico - namespace: tigera-operator - releaseName: projectcalico - repoURL: https://docs.tigera.io/calico/charts - valuesTemplate: |- - installation: - cni: - type: Calico - calicoNetwork: - bgp: Disabled - mtu: 1350 - ipPools: - ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} - - cidr: {{ $cidr }} - encapsulation: VXLAN{{end}} - registry: mcr.microsoft.com/oss - # Image and registry configuration for the tigera/operator pod. - tigeraOperator: - image: tigera/operator - registry: mcr.microsoft.com/oss - calicoctl: - image: mcr.microsoft.com/oss/calico/ctl - version: ${CALICO_VERSION} ---- -apiVersion: addons.cluster.x-k8s.io/v1alpha1 -kind: HelmChartProxy -metadata: - name: azuredisk-csi-driver-chart - namespace: default -spec: - chartName: azuredisk-csi-driver - clusterSelector: - matchLabels: - azuredisk-csi: "true" - namespace: kube-system - releaseName: azuredisk-csi-driver-oot - repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts - valuesTemplate: |- - controller: - replicas: 1 - runOnControlPlane: true - windows: - useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} ---- -apiVersion: addons.cluster.x-k8s.io/v1alpha1 -kind: HelmChartProxy -metadata: - name: cloud-provider-azure-chart - namespace: default -spec: - chartName: cloud-provider-azure - clusterSelector: - matchLabels: - cloud-provider: azure - releaseName: cloud-provider-azure-oot - repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo - valuesTemplate: | - infra: - clusterName: {{ .Cluster.metadata.name }} - cloudControllerManager: - clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} - logVerbosity: 4 ---- -apiVersion: addons.cluster.x-k8s.io/v1alpha1 -kind: HelmChartProxy -metadata: - name: cloud-provider-azure-chart-ci - namespace: default -spec: - chartName: cloud-provider-azure - clusterSelector: - matchLabels: - cloud-provider: azure-ci - releaseName: cloud-provider-azure-oot - repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo - valuesTemplate: | - infra: - clusterName: {{ .Cluster.metadata.name }} - cloudControllerManager: - cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} - cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} - clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} - imageName: "${CCM_IMAGE_NAME:-""}" - imageRepository: "${IMAGE_REGISTRY:-""}" - imageTag: "${IMAGE_TAG_CCM:-""}" - logVerbosity: ${CCM_LOG_VERBOSITY:-4} - replicas: ${CCM_COUNT:-1} - enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} - cloudNodeManager: - imageName: "${CNM_IMAGE_NAME:-""}" - imageRepository: "${IMAGE_REGISTRY:-""}" - imageTag: "${IMAGE_TAG_CNM:-""}" ---- -apiVersion: v1 -data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\nkind: ConfigMap\napiVersion: - v1\nmetadata:\n name: calico-config-windows\n namespace: calico-system\n labels:\n - \ tier: node\n app: calico\ndata:\n veth_mtu: \"1350\"\n \n cni_network_config: - |\n {\n \"name\": \"Calico\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": - [\n {\n \"windows_use_single_network\": true,\n \"type\": - \"calico\",\n \"mode\": \"vxlan\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n - \ \"nodename_file_optional\": true,\n \"log_file_path\": \"c:/cni.log\",\n - \ \"log_level\": \"debug\",\n\n \"vxlan_mac_prefix\": \"0E-2A\",\n - \ \"vxlan_vni\": 4096,\n \"mtu\": __CNI_MTU__,\n \"policy\": - {\n \"type\": \"k8s\"\n },\n\n \"log_level\": \"info\",\n\n - \ \"capabilities\": {\"dns\": true},\n \"DNS\": {\n \"Search\": - \ [\n \"svc.cluster.local\"\n ]\n },\n\n \"datastore_type\": - \"kubernetes\",\n\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n - \ },\n\n \"ipam\": {\n \"type\": \"calico-ipam\",\n - \ \"subnet\": \"usePodCidr\"\n },\n\n \"policies\": - \ [\n {\n \"Name\": \"EndpointPolicy\",\n \"Value\": - \ {\n \"Type\": \"OutBoundNAT\",\n \"ExceptionList\": - \ [\n \"__K8S_SERVICE_CIDR__\"\n ]\n }\n - \ },\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Value\": {\n \"Type\": \"SDNROUTE\",\n \"DestinationPrefix\": - \ \"__K8S_SERVICE_CIDR__\",\n \"NeedEncap\": true\n }\n - \ }\n ]\n }\n ]\n\n }\n---\napiVersion: apps/v1\nkind: - DaemonSet\nmetadata:\n name: calico-node-windows\n labels:\n tier: node\n - \ app: calico\n namespace: calico-system\nspec:\n selector:\n matchLabels:\n - \ app: calico\n template:\n metadata:\n labels:\n tier: node\n - \ app: calico\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - \ nodeSelectorTerms:\n - matchExpressions:\n - - key: kubernetes.io/os\n operator: In\n values:\n - \ - windows\n - key: kubernetes.io/arch\n - \ operator: In\n values:\n - - amd64\n securityContext:\n windowsOptions:\n hostProcess: - true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n hostNetwork: - true\n serviceAccountName: calico-node\n tolerations:\n - operator: - Exists\n effect: NoSchedule\n # Mark the pod as a critical add-on - for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n - \ - effect: NoExecute\n operator: Exists\n initContainers:\n # - This container installs the CNI binaries\n # and CNI network config file - on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.26.1-hostprocess\n - \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n - \ imagePullPolicy: Always\n env:\n # Name of the CNI - config file to create.\n - name: CNI_CONF_NAME\n value: - \"10-calico.conflist\"\n # The CNI network config to install on each - node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n - \ name: calico-config-windows\n key: cni_network_config\n - \ # Set the hostname based on the k8s node name.\n - name: - KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: - spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n - \ valueFrom:\n configMapKeyRef:\n name: - calico-config-windows\n key: veth_mtu\n # Prevents - the container from sleeping forever.\n - name: SLEEP\n value: - \"false\"\n - name: K8S_SERVICE_CIDR\n value: \"10.96.0.0/12\"\n - \ volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: - cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: - cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n - \ securityContext:\n windowsOptions:\n hostProcess: - true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n - \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.26.1-hostprocess\n - \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n - \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: - Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: - /etc/kube-calico-windows/\n env:\n - name: POD_NAME\n valueFrom:\n - \ fieldRef:\n apiVersion: v1\n fieldPath: - metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n - \ apiVersion: v1\n fieldPath: metadata.namespace\n - - name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n - \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n - \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n - \ image: sigwindowstools/calico-node:v3.26.1-hostprocess\n args: - [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: - Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n - \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n - \ - name: calico-static-rules\n mountPath: /calico/static-rules.json\n - \ subPath: static-rules.json\n env:\n - name: POD_NAME\n - \ valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: - metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n - \ apiVersion: v1\n fieldPath: metadata.namespace\n - - name: VXLAN_VNI\n value: \"4096\"\n - name: KUBECONFIG\n value: - \"C:/etc/cni/net.d/calico-kubeconfig\"\n volumes:\n - name: calico-config-windows\n - \ configMap:\n name: calico-config-windows\n - name: calico-static-rules\n - \ configMap:\n name: calico-static-rules\n # Used to install - CNI.\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n - \ - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n - \ - name: kubeadm-config\n configMap:\n name: kubeadm-config\n---\napiVersion: - apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n - \ group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: - IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: - false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n - \ properties:\n apiVersion:\n description: 'APIVersion - defines the versioned schema of this representation\n of an object. - Servers should convert recognized schemas to the latest\n internal - value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n - \ type: string\n kind:\n description: 'Kind is a - string value representing the REST resource this\n object represents. - Servers may infer this from the endpoint the client\n submits requests - to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n - \ type: string\n metadata:\n type: object\n spec:\n - \ description: IPAMConfigSpec contains the specification for an IPAMConfig\n - \ resource.\n properties:\n autoAllocateBlocks:\n - \ type: boolean\n maxBlocksPerHost:\n description: - MaxBlocksPerHost, if non-zero, is the max number of blocks\n that - can be affine to each host.\n maximum: 2147483647\n minimum: - 0\n type: integer\n strictAffinity:\n type: - boolean\n required:\n - autoAllocateBlocks\n - - strictAffinity\n type: object\n type: object\n served: true\n - \ storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n - \ conditions: []\n storedVersions: []\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: metrics-server-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - metrics-server: enabled - resources: - - kind: ConfigMap - name: metrics-server-${CLUSTER_NAME} - strategy: ApplyOnce ---- -apiVersion: v1 -data: - metrics-server: | - apiVersion: v1 - kind: ServiceAccount - metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - k8s-app: metrics-server - rbac.authorization.k8s.io/aggregate-to-admin: "true" - rbac.authorization.k8s.io/aggregate-to-edit: "true" - rbac.authorization.k8s.io/aggregate-to-view: "true" - name: system:aggregated-metrics-reader - rules: - - apiGroups: - - metrics.k8s.io - resources: - - pods - - nodes - verbs: - - get - - list - - watch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server - rules: - - apiGroups: - - "" - resources: - - nodes/metrics - verbs: - - get - - apiGroups: - - "" - resources: - - pods - - nodes - verbs: - - get - - list - - watch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - labels: - k8s-app: metrics-server - name: metrics-server-auth-reader - namespace: kube-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader - subjects: - - kind: ServiceAccount - name: metrics-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - labels: - k8s-app: metrics-server - name: metrics-server:system:auth-delegator - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator - subjects: - - kind: ServiceAccount - name: metrics-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:metrics-server - subjects: - - kind: ServiceAccount - name: metrics-server - namespace: kube-system - --- - apiVersion: v1 - kind: Service - metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system - spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - k8s-app: metrics-server - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: metrics-server - strategy: - rollingUpdate: - maxUnavailable: 0 - template: - metadata: - labels: - k8s-app: metrics-server - spec: - containers: - - args: - - --cert-dir=/tmp - - --secure-port=4443 - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --kubelet-use-node-status-port - - --metric-resolution=15s - - --kubelet-insecure-tls - image: registry.k8s.io/metrics-server/metrics-server:v0.6.3 - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 3 - httpGet: - path: /livez - port: https - scheme: HTTPS - periodSeconds: 10 - name: metrics-server - ports: - - containerPort: 4443 - name: https - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /readyz - port: https - scheme: HTTPS - initialDelaySeconds: 20 - periodSeconds: 10 - resources: - requests: - cpu: 100m - memory: 200Mi - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 1000 - volumeMounts: - - mountPath: /tmp - name: tmp-dir - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-cluster-critical - serviceAccountName: metrics-server - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - volumes: - - emptyDir: {} - name: tmp-dir - --- - apiVersion: apiregistration.k8s.io/v1 - kind: APIService - metadata: - labels: - k8s-app: metrics-server - name: v1beta1.metrics.k8s.io - spec: - group: metrics.k8s.io - groupPriorityMinimum: 100 - insecureSkipTLSVerify: true - service: - name: metrics-server - namespace: kube-system - version: v1beta1 - versionPriority: 100 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: metrics-server-${CLUSTER_NAME} - namespace: default ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-storageclass - namespace: default -spec: - clusterSelector: - matchLabels: - storageclass: "true" - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-storageclass - strategy: ApplyOnce ---- -apiVersion: v1 -data: - storageclass: | - apiVersion: storage.k8s.io/v1 - kind: StorageClass - metadata: - name: default - annotations: - storageclass.beta.kubernetes.io/is-default-class: "true" - labels: - kubernetes.io/cluster-service: "true" - provisioner: kubernetes.io/azure-disk - parameters: - kind: Managed - storageaccounttype: Standard_LRS - cachingmode: ReadOnly - volumeBindingMode: WaitForFirstConsumer - --- - apiVersion: storage.k8s.io/v1 - kind: StorageClass - metadata: - name: managed-premium - annotations: - labels: - kubernetes.io/cluster-service: "true" - provisioner: kubernetes.io/azure-disk - parameters: - kind: Managed - storageaccounttype: Premium_LRS - cachingmode: ReadOnly - volumeBindingMode: WaitForFirstConsumer - --- - apiVersion: storage.k8s.io/v1 - kind: StorageClass - metadata: - name: managed-standard - annotations: - labels: - kubernetes.io/cluster-service: "true" - provisioner: kubernetes.io/azure-disk - parameters: - kind: Managed - storageaccounttype: Standard_LRS - cachingmode: ReadOnly - volumeBindingMode: WaitForFirstConsumer -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: storageclass-${CLUSTER_NAME} - namespace: default diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index b2b0b302908a..1d07c5d7372d 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -1093,6 +1093,20 @@ spec: name: metrics-server-${CLUSTER_NAME} strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-storageclass + namespace: default +spec: + clusterSelector: + matchLabels: + storageclass: "true" + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-storageclass + strategy: ApplyOnce +--- apiVersion: v1 data: metrics-server: | @@ -1309,20 +1323,6 @@ metadata: name: metrics-server-${CLUSTER_NAME} namespace: default --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-storageclass - namespace: default -spec: - clusterSelector: - matchLabels: - storageclass: "true" - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-storageclass - strategy: ApplyOnce ---- apiVersion: v1 data: storageclass: | diff --git a/templates/test/dev/custom-builds-load/kustomization.yaml b/templates/test/dev/custom-builds-load/kustomization.yaml deleted file mode 100644 index ae9cf29326f2..000000000000 --- a/templates/test/dev/custom-builds-load/kustomization.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: default -resources: -- ../custom-builds -- storageclass-resource-set.yaml -patches: -- path: patches/cluster-label-storageclass.yaml -- path: patches/cluster-label-azuredisk.yaml -- path: patches/kcp-scheduler.yaml -configMapGenerator: -- files: - - storageclass=../../../addons/storageclass-azure-disk.yaml - name: storageclass-${CLUSTER_NAME} -generatorOptions: - annotations: - note: generated - disableNameSuffixHash: true - labels: - type: generated - -sortOptions: - order: fifo diff --git a/templates/test/dev/custom-builds/kustomization.yaml b/templates/test/dev/custom-builds/kustomization.yaml index f9651b84640f..2a670d2f9021 100644 --- a/templates/test/dev/custom-builds/kustomization.yaml +++ b/templates/test/dev/custom-builds/kustomization.yaml @@ -4,6 +4,7 @@ namespace: default resources: - ../../../test/ci/prow - ../../../addons/metrics-server/metrics-server-resource-set.yaml +- storageclass-resource-set.yaml patches: - path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider.yaml target: @@ -52,6 +53,9 @@ patches: - path: ../../../test/ci/patches/windows-machine-deployment-replicas.yaml - path: ../../../test/ci/patches/metrics-server-enabled-cluster.yaml - path: ../../../test/ci/patches/controller-manager-featuregates.yaml +- path: patches/cluster-label-storageclass.yaml +- path: patches/cluster-label-azuredisk.yaml +- path: patches/kcp-scheduler.yaml configMapGenerator: - behavior: merge files: @@ -60,6 +64,9 @@ configMapGenerator: - files: - metrics-server=../../../addons/metrics-server/metrics-server.yaml name: metrics-server-${CLUSTER_NAME} +- files: + - storageclass=../../../addons/storageclass-azure-disk.yaml + name: storageclass-${CLUSTER_NAME} generatorOptions: annotations: note: generated diff --git a/templates/test/dev/custom-builds-load/patches/cluster-label-azuredisk.yaml b/templates/test/dev/custom-builds/patches/cluster-label-azuredisk.yaml similarity index 100% rename from templates/test/dev/custom-builds-load/patches/cluster-label-azuredisk.yaml rename to templates/test/dev/custom-builds/patches/cluster-label-azuredisk.yaml diff --git a/templates/test/dev/custom-builds-load/patches/cluster-label-storageclass.yaml b/templates/test/dev/custom-builds/patches/cluster-label-storageclass.yaml similarity index 100% rename from templates/test/dev/custom-builds-load/patches/cluster-label-storageclass.yaml rename to templates/test/dev/custom-builds/patches/cluster-label-storageclass.yaml diff --git a/templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml b/templates/test/dev/custom-builds/patches/kcp-scheduler.yaml similarity index 100% rename from templates/test/dev/custom-builds-load/patches/kcp-scheduler.yaml rename to templates/test/dev/custom-builds/patches/kcp-scheduler.yaml diff --git a/templates/test/dev/custom-builds-load/storageclass-resource-set.yaml b/templates/test/dev/custom-builds/storageclass-resource-set.yaml similarity index 100% rename from templates/test/dev/custom-builds-load/storageclass-resource-set.yaml rename to templates/test/dev/custom-builds/storageclass-resource-set.yaml