From 0530b98b4471c610a7ef743088a0530df60c9364 Mon Sep 17 00:00:00 2001 From: Karthik K N Date: Mon, 25 Mar 2024 12:11:28 +0530 Subject: [PATCH] Support ignition --- api/v1beta2/ibmpowervscluster_types.go | 2 +- api/v1beta2/ibmpowervsmachine_types.go | 2 +- cloud/scope/powervs_machine.go | 35 ++ docs/book/src/SUMMARY.md | 1 + docs/book/src/developer/tilt.md | 19 + docs/book/src/topics/powervs/ignition.md | 30 ++ docs/book/src/topics/powervs/index.md | 1 + .../cluster-template-powervs-ignition.yaml | 489 ++++++++++++++++++ 8 files changed, 577 insertions(+), 2 deletions(-) create mode 100644 docs/book/src/topics/powervs/ignition.md create mode 100644 templates/cluster-template-powervs-ignition.yaml diff --git a/api/v1beta2/ibmpowervscluster_types.go b/api/v1beta2/ibmpowervscluster_types.go index b6595098d..a2c653572 100644 --- a/api/v1beta2/ibmpowervscluster_types.go +++ b/api/v1beta2/ibmpowervscluster_types.go @@ -142,7 +142,7 @@ type Ignition struct { // Version defines which version of Ignition will be used to generate bootstrap data. // // +optional - // +kubebuilder:default="2.3" + // +kubebuilder:default="2.4" // +kubebuilder:validation:Enum="2.3";"2.4";"3.0";"3.1";"3.2";"3.3";"3.4" Version string `json:"version,omitempty"` } diff --git a/api/v1beta2/ibmpowervsmachine_types.go b/api/v1beta2/ibmpowervsmachine_types.go index 300775108..3e9cb54e7 100644 --- a/api/v1beta2/ibmpowervsmachine_types.go +++ b/api/v1beta2/ibmpowervsmachine_types.go @@ -41,7 +41,7 @@ const ( // PowerVSProcessorTypeCapped enum property to identify a Capped Power VS processor type. PowerVSProcessorTypeCapped PowerVSProcessorType = "Capped" // DefaultIgnitionVersion represents default Ignition version generated for machine userdata. - DefaultIgnitionVersion = "2.3" + DefaultIgnitionVersion = "2.4" ) // IBMPowerVSMachineSpec defines the desired state of IBMPowerVSMachine. diff --git a/cloud/scope/powervs_machine.go b/cloud/scope/powervs_machine.go index 5a4bf5936..cf3e8ba13 100644 --- a/cloud/scope/powervs_machine.go +++ b/cloud/scope/powervs_machine.go @@ -357,6 +357,41 @@ func (m *PowerVSMachineScope) resolveUserData() (string, error) { if err != nil { return "", err } + + if getIgnitionVersion(m) == "2.4" { + var ignData ignV2Types.Config + if err := json.Unmarshal(userData, &ignData); err != nil { + m.Error(err, "error while unmarshalling ignition data") + return "", err + } + + // we make use of HTTPHeaders for fetching the ignition data from the COS bucket https://github.com/kubernetes-sigs/cluster-api-provider-ibmcloud/blob/258630acf8ef19c390b8defc64b4500c16316fe5/cloud/scope/powervs_machine.go#L470-L474 + // by default cluster api bootstrap controller will create ignition with 2.3.0 version, but that version dont have HTTPHeaders authentication(https://github.com/coreos/ignition/blob/spec2x/doc/configuration-v2_3.md) + // So we convert the version to 2.4.0 to make use of HTTPHeaders for authentication(https://github.com/coreos/ignition/blob/spec2x/doc/configuration-v2_4.md) + if ignData.Ignition.Version == "2.3.0" { + ignData.Ignition.Version = "2.4.0" + } + + // TODO(RemoveThis): For debug only + //ignData.Passwd = ignV2Types.Passwd{ + // Users: []ignV2Types.PasswdUser{ + // { + // Name: "", + // PasswordHash: core.StringPtr(""), + // SSHAuthorizedKeys: []ignV2Types.SSHAuthorizedKey{ + // "", + // }, + // }, + // }, + //} + + userData, err = json.Marshal(ignData) + if err != nil { + m.Error(err, "error while marshalling ignition data") + return "", err + } + } + if m.UseIgnition(userDataFormat) { data, err := m.ignitionUserData(userData) if err != nil { diff --git a/docs/book/src/SUMMARY.md b/docs/book/src/SUMMARY.md index 899ccc543..21c90db47 100644 --- a/docs/book/src/SUMMARY.md +++ b/docs/book/src/SUMMARY.md @@ -18,6 +18,7 @@ - [Creating a cluster with External Cloud Provider](./topics/powervs/external-cloud-provider.md) - [Creating a cluster from ClusterClass](./topics/powervs/clusterclass-cluster.md) - [Creating a cluster by auto creating required resources](./topics/powervs/create-resources.md) + - [Creating a cluster using ignition](./topics/powervs/ignition.md) - [Using autoscaler with scaling from 0 machine](./topics/powervs/autoscaler-scalling-from-0.md) - [capibmadm CLI](./topics/capibmadm/index.md) - [PowerVS Commands](./topics/capibmadm/powervs/index.md) diff --git a/docs/book/src/developer/tilt.md b/docs/book/src/developer/tilt.md index bf0a411d3..9c4a2ff42 100644 --- a/docs/book/src/developer/tilt.md +++ b/docs/book/src/developer/tilt.md @@ -155,6 +155,25 @@ kustomize_substitutions: IBMCLOUD_AUTH_URL: "https://iam.test.cloud.ibm.com" ``` +### 4. Configuration to deploy PowerVS workload cluster with ignition + +Set ```EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true"``` to update cluster-api bootstrap provider to use ignition format. + +```yaml +default_registry: "gcr.io/you-project-name-here" +provider_repos: +- ../cluster-api-provider-ibmcloud +enable_providers: +- ibmcloud +- kubeadm-bootstrap +- kubeadm-control-plane +kustomize_substitutions: + IBMCLOUD_API_KEY: "XXXXXXXXXXXXXXXXXX" + PROVIDER_ID_FORMAT: "v2" + EXP_CLUSTER_RESOURCE_SET: "true" + EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true", +``` + **NOTE**: For information about all the fields that can be used in the `tilt-settings.yaml` file, check them [here](https://cluster-api.sigs.k8s.io/developer/tilt.html#tilt-settings-fields). ## Run Tilt diff --git a/docs/book/src/topics/powervs/ignition.md b/docs/book/src/topics/powervs/ignition.md new file mode 100644 index 000000000..14df620bd --- /dev/null +++ b/docs/book/src/topics/powervs/ignition.md @@ -0,0 +1,30 @@ +# Use ignition for IBM PowerVS cluster + +## Steps + +- Set ```powervs.cluster.x-k8s.io/create-infra:true``` annotation to IBMPowerVSCluster resource to auto create required resources. +- The cluster will be configured with IBM PowerVS external [cloud provider](https://kubernetes.io/docs/concepts/architecture/cloud-controller/) +- The [cluster-template-powervs-ignition](https://github.com/kubernetes-sigs/cluster-api-provider-ibmcloud/blob/main/templates/cluster-template-powervs-ignition.yaml) will use [clusterresourceset](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-resource-set.html) and will create the necessary config map, secret and roles to run the cloud controller manager +- As a prerequisite set the `provider-id-fmt` [flag](https://github.com/kubernetes-sigs/cluster-api-provider-ibmcloud/blob/5e7f80878f2252c6ab13c16102de90c784a2624d/main.go#L168-L173) with value v2. +- Set ```export EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION=true``` if using clusterctl or set ```"EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION": "true"``` when using tilt to update capi bootstrap provided to set ignition format. + +### Deploy PowerVS cluster with IBM PowerVS cloud provider + + ``` +IBMCLOUD_API_KEY=> \ +IBMPOWERVS_SSHKEY_NAME="karthik-ssh" \ +COS_BUCKET_REGION="us-south" \ +COS_BUCKET_NAME="power-oss-bucket" \ +COS_OBJECT_NAME=capi-rhcos-openstack-4.5.ova.gz \ +IBMACCOUNT_ID="" \ +IBMPOWERVS_REGION="wdc" \ +IBMPOWERVS_ZONE="wdc06" \ +IBMVPC_REGION="us-east" \ +IBM_RESOURCE_GROUP="ibm-hypershift-dev" \ +BASE64_API_KEY=$(echo -n $IBMCLOUD_API_KEY | base64) \ +clusterctl generate cluster capi-powervs- --kubernetes-version v1.28.4 \ +--target-namespace default \ +--control-plane-machine-count=3 \ +--worker-machine-count=1 \ +--from ./cluster-template-powervs-ignition.yaml | kubectl apply -f - + ``` diff --git a/docs/book/src/topics/powervs/index.md b/docs/book/src/topics/powervs/index.md index 567fe8394..c1ede47a2 100644 --- a/docs/book/src/topics/powervs/index.md +++ b/docs/book/src/topics/powervs/index.md @@ -6,4 +6,5 @@ - [Creating a cluster with external cloud provider](/topics/powervs/external-cloud-provider.html) - [Creating a cluster from ClusterClass](/topics/powervs/clusterclass-cluster.html) - [Creating a cluster by auto creating required resources](/topics/powervs/create-resources.html) +- [Creating a cluster using ignition](/topics/powervs/ignition.html) - [Using autoscaler with scaling from 0 machine](/topics/powervs/autoscaler-scalling-from-0.html) \ No newline at end of file diff --git a/templates/cluster-template-powervs-ignition.yaml b/templates/cluster-template-powervs-ignition.yaml new file mode 100644 index 000000000..c74e79750 --- /dev/null +++ b/templates/cluster-template-powervs-ignition.yaml @@ -0,0 +1,489 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + ccm: external + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMPowerVSCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMPowerVSCluster +metadata: + annotations: + powervs.cluster.x-k8s.io/create-infra: "true" + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} + namespace: default +spec: + resourceGroup: + name: ${IBM_RESOURCE_GROUP} + zone: ${IBMPOWERVS_ZONE} + serviceInstance: + name: ${CLUSTER_NAME}-serviceInstance + vpc: + name: ${CLUSTER_NAME}-vpc + region: ${IBMVPC_REGION} + vpcSubnets: + - name: ${CLUSTER_NAME}-vpcsubnet + transitGateway: + name: ${CLUSTER_NAME}-transitgateway + loadBalancers: + - name: ${CLUSTER_NAME}-loadbalancer + ignition: + version: "2.4" +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + controllerManager: + extraArgs: + cloud-provider: external + enable-hostpath-provisioner: "true" + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + systemd: + units: + - name: kubeadm.service + enabled: true + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + name: '{{ v1.local_hostname }}' + joinConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + name: '{{ v1.local_hostname }}' + preKubeadmCommands: + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname + - curl -N https://cdn.dl.k8s.io/release/v1.18.2/bin/linux/ppc64le/kubeadm -o kubeadm + - chmod +x kubeadm + - mkdir -p .local/bin/ + - cp -a kubeadm .local/bin/ + - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp + - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMPowerVSMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMPowerVSMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + imageRef: + name: ${CLUSTER_NAME}-image + memoryGiB: ${IBMPOWERVS_CONTROL_PLANE_MEMORY:=4} + processorType: ${IBMPOWERVS_CONTROL_PLANE_PROCTYPE:="Shared"} + processors: ${IBMPOWERVS_CONTROL_PLANE_PROCESSORS:="0.25"} + sshKey: ${IBMPOWERVS_SSHKEY_NAME} + systemType: ${IBMPOWERVS_CONTROL_PLANE_SYSTYPE:="s922"} + serviceInstance: + name: ${CLUSTER_NAME}-serviceInstance +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: IBMPowerVSMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMPowerVSMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + imageRef: + name: ${CLUSTER_NAME}-image + memoryGiB: ${IBMPOWERVS_COMPUTE_MEMORY:=4} + processorType: ${IBMPOWERVS_COMPUTE_PROCTYPE:="Shared"} + processors: ${IBMPOWERVS_COMPUTE_PROCESSORS:="0.25"} + sshKey: ${IBMPOWERVS_SSHKEY_NAME} + systemType: ${IBMPOWERVS_COMPUTE_SYSTYPE:="s922"} + serviceInstance: + name: ${CLUSTER_NAME}-serviceInstance +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + cluster.x-k8s.io/control-plane: "" + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + systemd: + units: + - name: kubeadm.service + enabled: true + joinConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + name: '{{ v1.local_hostname }}' + preKubeadmCommands: + - hostname "{{ v1.local_hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts + - echo "{{ v1.local_hostname }}" >/etc/hostname + - curl -N https://cdn.dl.k8s.io/release/v1.18.2/bin/linux/ppc64le/kubeadm -o kubeadm + - chmod +x kubeadm + - mkdir -p .local/bin/ + - cp -a kubeadm .local/bin/ + - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp + - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: IBMPowerVSImage +metadata: + creationTimestamp: null + name: ${CLUSTER_NAME}-image +spec: + bucket: ${COS_BUCKET_NAME} + clusterName: ${CLUSTER_NAME} + deletePolicy: delete + object: ${COS_OBJECT_NAME} + region: ${COS_BUCKET_REGION} + serviceInstance: + name: ${CLUSTER_NAME}-serviceInstance +status: + ready: false +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-cloud-conf + namespace: default +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: Secret + name: ibmpowervs-credential + - kind: ConfigMap + name: ibmpowervs-cfg + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + ibmpowervs-cloud-conf.yaml: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: ibmpowervs-cloud-config + namespace: kube-system + data: + ibmpowervs.conf: | + [global] + version = 1.1.0 + [kubernetes] + config-file = "" + [provider] + cluster-default-provider = g2 + accountID = ${IBMACCOUNT_ID} + clusterID = ${CLUSTER_NAME} + g2workerServiceAccountID = ${IBMACCOUNT_ID} + g2Credentials = /etc/ibm-secret/ibmcloud_api_key + g2ResourceGroupName = ${IBM_RESOURCE_GROUP:=""} + g2VpcSubnetNames = ${CLUSTER_NAME}-vpcsubnet + g2VpcName = ${CLUSTER_NAME}-vpc + region = ${IBMVPC_REGION:=""} + powerVSRegion = ${IBMPOWERVS_REGION} + powerVSZone = ${IBMPOWERVS_ZONE} + powerVSCloudInstanceName = ${CLUSTER_NAME}-serviceInstance +kind: ConfigMap +metadata: + name: ibmpowervs-cfg + namespace: default +--- +apiVersion: v1 +kind: Secret +metadata: + name: ibmpowervs-credential + namespace: default +stringData: + ibmpowervs-credential.yaml: |- + apiVersion: v1 + kind: Secret + metadata: + name: ibmpowervs-cloud-credential + namespace: kube-system + data: + ibmcloud_api_key: ${BASE64_API_KEY} +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +data: + ibmpowervs-ccm-external.yaml: |- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resourceNames: + - node-controller + - service-controller + resources: + - serviceaccounts/token + verbs: + - create + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: ibmpowervs-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: ibmpowervs-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: ibmpowervs-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: ibmpowervs-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + operator: Exists + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + operator: Exists + - key: node.kubernetes.io/not-ready + effect: NoSchedule + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: ibmpowervs-cloud-controller-manager + image: gcr.io/k8s-staging-capi-ibmcloud/powervs-cloud-controller-manager:6c98ec5 + args: + - --v=2 + - --cloud-provider=ibm + - --cloud-config=/etc/cloud/ibmpowervs.conf + - --use-service-account-credentials=true + env: + - name: ENABLE_VPC_PUBLIC_ENDPOINT + value: "true" + volumeMounts: + - mountPath: /etc/cloud + name: ibmpowervs-config-volume + readOnly: true + - mountPath: /etc/ibm-secret + name: ibm-secret + resources: + requests: + cpu: 200m + hostNetwork: true + volumes: + - name: ibmpowervs-config-volume + configMap: + name: ibmpowervs-cloud-config + - name: ibm-secret + secret: + secretName: ibmpowervs-cloud-credential +kind: ConfigMap +metadata: + name: cloud-controller-manager-addon + namespace: default