From 30691e0fe1b01d7dd83e85b6f4f5e9c4cf586e03 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Tue, 28 Mar 2023 18:58:32 +1100 Subject: [PATCH 01/20] feat: Add manila csi In addition, deploy cinder-csi plugin as conditional --- magnum_cluster_api/clients.py | 39 +++- magnum_cluster_api/conf.py | 52 ++++- .../cinder-csi-controllerplugin-rbac.yaml | 0 .../cinder-csi-controllerplugin.yaml | 0 .../cinder-csi-nodeplugin-rbac.yaml | 0 .../cinder-csi-nodeplugin.yaml | 0 .../csi-cinder-driver.yaml | 0 .../manila-csi/csi-controllerplugin-rbac.yaml | 113 ++++++++++ .../manila-csi/csi-controllerplugin.yaml | 128 +++++++++++ .../manila-csi/csi-nodeplugin-rbac.yaml | 58 +++++ .../manifests/manila-csi/csi-nodeplugin.yaml | 98 +++++++++ .../manifests/manila-csi/csidriver.yaml | 8 + magnum_cluster_api/resources.py | 206 +++++++++++------- magnum_cluster_api/utils.py | 56 +++++ 14 files changed, 680 insertions(+), 78 deletions(-) rename magnum_cluster_api/manifests/{csi => cinder-csi}/cinder-csi-controllerplugin-rbac.yaml (100%) rename magnum_cluster_api/manifests/{csi => cinder-csi}/cinder-csi-controllerplugin.yaml (100%) rename magnum_cluster_api/manifests/{csi => cinder-csi}/cinder-csi-nodeplugin-rbac.yaml (100%) rename magnum_cluster_api/manifests/{csi => cinder-csi}/cinder-csi-nodeplugin.yaml (100%) rename magnum_cluster_api/manifests/{csi => cinder-csi}/csi-cinder-driver.yaml (100%) create mode 100644 magnum_cluster_api/manifests/manila-csi/csi-controllerplugin-rbac.yaml create mode 100644 magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml create mode 100644 magnum_cluster_api/manifests/manila-csi/csi-nodeplugin-rbac.yaml create mode 100644 magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml create mode 100644 magnum_cluster_api/manifests/manila-csi/csidriver.yaml diff --git a/magnum_cluster_api/clients.py b/magnum_cluster_api/clients.py index eb609b06..b831d69f 100644 --- a/magnum_cluster_api/clients.py +++ b/magnum_cluster_api/clients.py @@ -13,7 +13,40 @@ # under the License. import pykube -from magnum.common import clients +from magnum.common import clients, exception +from manilaclient.v2 import client as manilaclient + + +class OpenStackClients(clients.OpenStackClients): + """Convenience class to create and cache client instances.""" + + def __init__(self, context): + super(OpenStackClients, self).__init__(context) + self._manila = None + + @exception.wrap_keystone_exception + def manila(self): + if self._manila: + return self._manila + endpoint_type = self._get_client_option("manila", "endpoint_type") + region_name = self._get_client_option("manila", "region_name") + manilaclient_version = self._get_client_option("manila", "api_version") + endpoint = self.url_for( + service_type="sharev2", interface=endpoint_type, region_name=region_name + ) + args = { + "cacert": self._get_client_option("manila", "ca_file"), + "insecure": self._get_client_option("manila", "insecure"), + } + + session = self.keystone().session + self._manila = manilaclient.Client( + api_version=manilaclient_version, + session=session, + service_catalog_url=endpoint, + **args + ) + return self._manila def get_pykube_api() -> pykube.HTTPClient: @@ -22,3 +55,7 @@ def get_pykube_api() -> pykube.HTTPClient: def get_openstack_api(context) -> clients.OpenStackClients: return clients.OpenStackClients(context) + + +def get_openstack_api(context) -> OpenStackClients: + return OpenStackClients(context) diff --git a/magnum_cluster_api/conf.py b/magnum_cluster_api/conf.py index f77bc3b3..a37e18e1 100644 --- a/magnum_cluster_api/conf.py +++ b/magnum_cluster_api/conf.py @@ -12,8 +12,16 @@ # License for the specific language governing permissions and limitations # under the License. +from magnum.i18n import _ from oslo_config import cfg +auto_scaling_group = cfg.OptGroup(name="auto_scaling", title="Options for auto scaling") + +manila_client_group = cfg.OptGroup( + name="manila_client", title="Options for the Manila client" +) + + auto_scaling_opts = [ cfg.StrOpt( "image_repository", @@ -47,5 +55,47 @@ ), ] + +manila_client_opts = [ + cfg.StrOpt( + "region_name", + help=_( + "Region in Identity service catalog to use for " + "communication with the OpenStack service." + ), + ), + cfg.StrOpt( + "endpoint_type", + default="publicURL", + help=_( + "Type of endpoint in Identity service catalog to use " + "for communication with the OpenStack service." + ), + ), + cfg.StrOpt( + "api_version", + default="3", + help=_("Version of Manila API to use in manilaclient."), + ), +] + +common_security_opts = [ + cfg.StrOpt("ca_file", help=_("Optional CA cert file to use in SSL connections.")), + cfg.StrOpt("cert_file", help=_("Optional PEM-formatted certificate chain file.")), + cfg.StrOpt( + "key_file", + help=_("Optional PEM-formatted file that contains the " "private key."), + ), + cfg.BoolOpt( + "insecure", + default=False, + help=_("If set, then the server's certificate will not " "be verified."), + ), +] + CONF = cfg.CONF -CONF.register_opts(auto_scaling_opts, "auto_scaling") +CONF.register_group(auto_scaling_group) +CONF.register_group(manila_client_group) +CONF.register_opts(auto_scaling_opts, group=auto_scaling_group) +CONF.register_opts(manila_client_opts, group=manila_client_group) +CONF.register_opts(manila_client_opts, group=common_security_opts) diff --git a/magnum_cluster_api/manifests/csi/cinder-csi-controllerplugin-rbac.yaml b/magnum_cluster_api/manifests/cinder-csi/cinder-csi-controllerplugin-rbac.yaml similarity index 100% rename from magnum_cluster_api/manifests/csi/cinder-csi-controllerplugin-rbac.yaml rename to magnum_cluster_api/manifests/cinder-csi/cinder-csi-controllerplugin-rbac.yaml diff --git a/magnum_cluster_api/manifests/csi/cinder-csi-controllerplugin.yaml b/magnum_cluster_api/manifests/cinder-csi/cinder-csi-controllerplugin.yaml similarity index 100% rename from magnum_cluster_api/manifests/csi/cinder-csi-controllerplugin.yaml rename to magnum_cluster_api/manifests/cinder-csi/cinder-csi-controllerplugin.yaml diff --git a/magnum_cluster_api/manifests/csi/cinder-csi-nodeplugin-rbac.yaml b/magnum_cluster_api/manifests/cinder-csi/cinder-csi-nodeplugin-rbac.yaml similarity index 100% rename from magnum_cluster_api/manifests/csi/cinder-csi-nodeplugin-rbac.yaml rename to magnum_cluster_api/manifests/cinder-csi/cinder-csi-nodeplugin-rbac.yaml diff --git a/magnum_cluster_api/manifests/csi/cinder-csi-nodeplugin.yaml b/magnum_cluster_api/manifests/cinder-csi/cinder-csi-nodeplugin.yaml similarity index 100% rename from magnum_cluster_api/manifests/csi/cinder-csi-nodeplugin.yaml rename to magnum_cluster_api/manifests/cinder-csi/cinder-csi-nodeplugin.yaml diff --git a/magnum_cluster_api/manifests/csi/csi-cinder-driver.yaml b/magnum_cluster_api/manifests/cinder-csi/csi-cinder-driver.yaml similarity index 100% rename from magnum_cluster_api/manifests/csi/csi-cinder-driver.yaml rename to magnum_cluster_api/manifests/cinder-csi/csi-cinder-driver.yaml diff --git a/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin-rbac.yaml b/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin-rbac.yaml new file mode 100644 index 00000000..cbf3c601 --- /dev/null +++ b/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin-rbac.yaml @@ -0,0 +1,113 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: openstack-manila-csi-controllerplugin + labels: + app: openstack-manila-csi + component: controllerplugin +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openstack-manila-csi-controllerplugin + labels: + app: openstack-manila-csi + component: controllerplugin +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-controllerplugin: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openstack-manila-csi-controllerplugin-rules + labels: + app: openstack-manila-csi + component: controllerplugin + rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-controllerplugin: "true" +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openstack-manila-csi-controllerplugin + labels: + app: openstack-manila-csi + component: controllerplugin +subjects: + - kind: ServiceAccount + name: openstack-manila-csi-controllerplugin + namespace: default +roleRef: + kind: ClusterRole + name: openstack-manila-csi-controllerplugin + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openstack-manila-csi-controllerplugin + labels: + app: openstack-manila-csi + component: controllerplugin +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openstack-manila-csi-controllerplugin + labels: + app: openstack-manila-csi + component: controllerplugin +subjects: + - kind: ServiceAccount + name: openstack-manila-csi-controllerplugin + namespace: default +roleRef: + kind: Role + name: openstack-manila-csi-controllerplugin + apiGroup: rbac.authorization.k8s.io diff --git a/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml b/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml new file mode 100644 index 00000000..e875e055 --- /dev/null +++ b/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml @@ -0,0 +1,128 @@ +kind: Service +apiVersion: v1 +metadata: + name: openstack-manila-csi-controllerplugin + labels: + app: openstack-manila-csi + component: controllerplugin +spec: + selector: + app: openstack-manila-csi + component: controllerplugin + ports: + - name: dummy + port: 12345 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: openstack-manila-csi-controllerplugin + labels: + app: openstack-manila-csi + component: controllerplugin +spec: + serviceName: openstack-manila-csi-controllerplugin + replicas: 1 + selector: + matchLabels: + app: openstack-manila-csi + component: controllerplugin + template: + metadata: + labels: + app: openstack-manila-csi + component: controllerplugin + spec: + serviceAccountName: openstack-manila-csi-controllerplugin + containers: + - name: provisioner + image: "registry.k8s.io/sig-storage/csi-provisioner:v3.0.0" + args: + - "--csi-address=$(ADDRESS)" + # To enable topology awareness in csi-provisioner, uncomment the following line: + # - "--feature-gates=Topology=true" + env: + - name: ADDRESS + value: "unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi-controllerplugin.sock" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org + - name: snapshotter + image: "registry.k8s.io/sig-storage/csi-snapshotter:v5.0.1" + args: + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: "unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi-controllerplugin.sock" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org + - name: resizer + image: "registry.k8s.io/sig-storage/csi-resizer:v1.3.0" + args: + - "--csi-address=$(ADDRESS)" + - "--handle-volume-inuse-error=false" + env: + - name: ADDRESS + value: "unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi-controllerplugin.sock" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org + - name: nodeplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: registry.k8s.io/provider-os/manila-csi-plugin:v1.27.0-alpha.0 + command: ["/bin/sh", "-c", + '/bin/manila-csi-plugin + --nodeid=$(NODE_ID) + --endpoint=$(CSI_ENDPOINT) + --drivername=$(DRIVER_NAME) + --share-protocol-selector=$(MANILA_SHARE_PROTO) + --fwdendpoint=$(FWD_CSI_ENDPOINT)' + # To enable topology awareness and retrieve compute node AZs from the OpenStack Metadata Service, add the following flags: + # --with-topology + # --nodeaz=$(curl http://169.254.169.254/openstack/latest/meta_data.json | jq -r .availability_zone) + # Those flags need to be added to csi-nodeplugin.yaml as well. + ] + env: + - name: DRIVER_NAME + value: manila.csi.openstack.org + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: "unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi-controllerplugin.sock" + - name: FWD_CSI_ENDPOINT + value: "unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock" + - name: MANILA_SHARE_PROTO + value: "NFS" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org + - name: fwd-plugin-dir + mountPath: /var/lib/kubelet/plugins/csi-nfsplugin + - name: pod-mounts + mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + volumes: + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/manila.csi.openstack.org + type: DirectoryOrCreate + - name: fwd-plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-nfsplugin + type: Directory + - name: pod-mounts + hostPath: + path: /var/lib/kubelet/pods + type: Directory + diff --git a/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin-rbac.yaml b/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin-rbac.yaml new file mode 100644 index 00000000..6d6228cd --- /dev/null +++ b/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin-rbac.yaml @@ -0,0 +1,58 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: openstack-manila-csi-nodeplugin + labels: + app: openstack-manila-csi + component: nodeplugin +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openstack-manila-csi-nodeplugin + labels: + app: openstack-manila-csi + component: nodeplugin +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-nodeplugin: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openstack-manila-csi-nodeplugin-rules + labels: + app: openstack-manila-csi + component: nodeplugin + rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-nodeplugin: "true" +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openstack-manila-csi-nodeplugin + labels: + app: openstack-manila-csi + component: nodeplugin +subjects: + - kind: ServiceAccount + name: openstack-manila-csi-nodeplugin + namespace: default +roleRef: + kind: ClusterRole + name: openstack-manila-csi-nodeplugin + apiGroup: rbac.authorization.k8s.io diff --git a/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml b/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml new file mode 100644 index 00000000..8bf4b3c7 --- /dev/null +++ b/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml @@ -0,0 +1,98 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: openstack-manila-csi-nodeplugin + labels: + app: openstack-manila-csi + component: nodeplugin +spec: + selector: + matchLabels: + app: openstack-manila-csi + component: nodeplugin + template: + metadata: + labels: + app: openstack-manila-csi + component: nodeplugin + spec: + serviceAccountName: openstack-manila-csi-nodeplugin + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: registrar + image: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.4.0" + args: + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/manila.csi.openstack.org/csi.sock" + lifecycle: + preStop: + exec: + command: [ + "/bin/sh", "-c", + 'rm -rf /registration/manila.csi.openstack.org + /registration/manila.csi.openstack.org-reg.sock' + ] + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + imagePullPolicy: IfNotPresent + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: nodeplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: registry.k8s.io/provider-os/manila-csi-plugin:latest + command: ["/bin/sh", "-c", + '/bin/manila-csi-plugin + --nodeid=$(NODE_ID) + --endpoint=$(CSI_ENDPOINT) + --drivername=$(DRIVER_NAME) + --share-protocol-selector=$(MANILA_SHARE_PROTO) + --fwdendpoint=$(FWD_CSI_ENDPOINT)' + # To enable topology awareness and retrieve compute node AZs from the OpenStack Metadata Service, add the following flags: + # --with-topology + # --nodeaz=$(curl http://169.254.169.254/openstack/latest/meta_data.json | jq -r .availability_zone) + # Those flags need to be added to csi-controllerplugin.yaml as well. + ] + env: + - name: DRIVER_NAME + value: manila.csi.openstack.org + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: "unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi.sock" + - name: FWD_CSI_ENDPOINT + value: "unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock" + - name: MANILA_SHARE_PROTO + value: "NFS" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org + - name: fwd-plugin-dir + mountPath: /var/lib/kubelet/plugins/csi-nfsplugin + volumes: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/manila.csi.openstack.org + type: DirectoryOrCreate + - name: fwd-plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-nfsplugin + type: DirectoryOrCreate + diff --git a/magnum_cluster_api/manifests/manila-csi/csidriver.yaml b/magnum_cluster_api/manifests/manila-csi/csidriver.yaml new file mode 100644 index 00000000..5d95ecd8 --- /dev/null +++ b/magnum_cluster_api/manifests/manila-csi/csidriver.yaml @@ -0,0 +1,8 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: manila.csi.openstack.org +spec: + attachRequired: false + podInfoOnMount: false + diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index 5ea5cde0..3d1fdeaf 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -35,7 +35,8 @@ CONF = cfg.CONF CLOUD_PROVIDER_TAG = "v1.25.3" CALICO_TAG = "v3.24.2" -CSI_TAG = "v1.25.3" +CINDER_CSI_TAG = "v1.25.3" +MANILA_CSI_TAG = "v1.25.3" CLUSTER_CLASS_VERSION = pkg_resources.require("magnum_cluster_api")[0].version CLUSTER_CLASS_NAME = f"magnum-v{CLUSTER_CLASS_VERSION}" @@ -163,15 +164,129 @@ def get_object(self) -> pykube.ConfigMap: ccm_version = utils.get_cluster_label( self.cluster, "cloud_provider_tag", CLOUD_PROVIDER_TAG ) - csi_version = utils.get_cluster_label( - self.cluster, "cinder_csi_plugin_tag", CSI_TAG - ) repository = utils.get_cluster_container_infra_prefix(self.cluster) osc = clients.get_openstack_api(self.context) - volume_types = osc.cinder().volume_types.list() - default_volume_type = osc.cinder().volume_types.default() + + data = { + **{ + os.path.basename(manifest): image_utils.update_manifest_images( + self.cluster.uuid, + manifest, + repository=repository, + replacements=[ + ( + "docker.io/k8scloudprovider/openstack-cloud-controller-manager:latest", + f"docker.io/k8scloudprovider/openstack-cloud-controller-manager:{ccm_version}", + ), + ], + ) + for manifest in glob.glob(os.path.join(manifests_path, "ccm/*.yaml")) + }, + **{ + "calico.yml": image_utils.update_manifest_images( + self.cluster.uuid, + os.path.join(manifests_path, f"calico/{calico_version}.yaml"), + repository=repository, + ) + }, + } + + if utils.is_cinder_csi_enabled(self.cluster): + volume_types = osc.cinder().volume_types.list() + default_volume_type = osc.cinder().volume_types.default() + csi_version = utils.get_cluster_label( + self.cluster, "cinder_csi_plugin_tag", CINDER_CSI_TAG + ) + data = { + **data, + **{ + os.path.basename(manifest): image_utils.update_manifest_images( + self.cluster.uuid, + manifest, + repository=repository, + replacements=[ + ( + "docker.io/k8scloudprovider/cinder-csi-plugin:latest", + f"docker.io/k8scloudprovider/cinder-csi-plugin:{csi_version}", + ), + ], + ) + for manifest in glob.glob( + os.path.join(manifests_path, "cinder-csi/*.yaml") + ) + }, + **{ + f"storageclass-cinder-{vt.name}.yaml": yaml.dump( + { + "apiVersion": objects.StorageClass.version, + "allowVolumeExpansion": True, + "kind": objects.StorageClass.kind, + "metadata": { + "annotations": { + "storageclass.kubernetes.io/is-default-class": "true" + } + if default_volume_type.name == vt.name + else {}, + "name": "cinder-%s" % vt.name.lower(), + }, + "provisioner": "kubernetes.io/cinder", + "parameters": { + "type": vt.name, + }, + "reclaimPolicy": "Delete", + "volumeBindingMode": "Immediate", + } + ) + for vt in volume_types + if vt.name != "__DEFAULT__" + }, + } + + if utils.is_manila_csi_enabled(self.cluster): + share_types = osc.manila().share_types.list() + csi_version = utils.get_cluster_label( + self.cluster, "manila_csi_plugin_tag", MANILA_CSI_TAG + ) + data = { + **data, + **{ + os.path.basename(manifest): image_utils.update_manifest_images( + self.cluster.uuid, + manifest, + repository=repository, + replacements=[ + ( + "registry.k8s.io/provider-os/manila-csi-plugin:latest", + f"registry.k8s.io/provider-os/manila-csi-plugin:{csi_version}", + ), + ], + ) + for manifest in glob.glob( + os.path.join(manifests_path, "manila-csi/*.yaml") + ) + }, + **{ + f"storageclass-manila-{st.name}.yaml": yaml.dump( + { + "apiVersion": objects.StorageClass.version, + "allowVolumeExpansion": True, + "kind": objects.StorageClass.kind, + "metadata": { + "name": "manila-%s" % st.name.lower(), + }, + "provisioner": "manila.csi.openstack.org", + "parameters": { + "type": st.name, + }, + "reclaimPolicy": "Delete", + "volumeBindingMode": "Immediate", + } + ) + for st in share_types + }, + } return pykube.ConfigMap( self.api, @@ -182,74 +297,7 @@ def get_object(self) -> pykube.ConfigMap: "name": self.cluster.uuid, "namespace": "magnum-system", }, - "data": { - **{ - os.path.basename(manifest): image_utils.update_manifest_images( - self.cluster.uuid, - manifest, - repository=repository, - replacements=[ - ( - "docker.io/k8scloudprovider/openstack-cloud-controller-manager:latest", - f"docker.io/k8scloudprovider/openstack-cloud-controller-manager:{ccm_version}", - ), - ], - ) - for manifest in glob.glob( - os.path.join(manifests_path, "ccm/*.yaml") - ) - }, - **{ - os.path.basename(manifest): image_utils.update_manifest_images( - self.cluster.uuid, - manifest, - repository=repository, - replacements=[ - ( - "docker.io/k8scloudprovider/cinder-csi-plugin:latest", - f"docker.io/k8scloudprovider/cinder-csi-plugin:{csi_version}", - ), - ], - ) - for manifest in glob.glob( - os.path.join(manifests_path, "csi/*.yaml") - ) - }, - **{ - "calico.yml": image_utils.update_manifest_images( - self.cluster.uuid, - os.path.join( - manifests_path, f"calico/{calico_version}.yaml" - ), - repository=repository, - ) - }, - **{ - f"storageclass-{vt.name}.yaml": yaml.dump( - { - "apiVersion": objects.StorageClass.version, - "allowVolumeExpansion": True, - "kind": objects.StorageClass.kind, - "metadata": { - "annotations": { - "storageclass.kubernetes.io/is-default-class": "true" - } - if default_volume_type.name == vt.name - else {}, - "name": vt.name.lower(), - }, - "provisioner": "kubernetes.io/cinder", - "parameters": { - "type": vt.name, - }, - "reclaimPolicy": "Delete", - "volumeBindingMode": "Immediate", - } - ) - for vt in volume_types - if vt.name != "__DEFAULT__" - }, - }, + "data": data, }, ) @@ -1334,13 +1382,19 @@ def labels(self) -> dict: "ccm": f"openstack-cloud-controller-manager-{ccm_version}", } - if utils.get_cluster_label_as_bool(self.cluster, "cinder_csi_enabled", True): + if utils.is_cinder_csi_enabled(self.cluster): csi_version = utils.get_cluster_label( - self.cluster, "cinder_csi_plugin_tag", CSI_TAG + self.cluster, "cinder_csi_plugin_tag", CINDER_CSI_TAG ) labels["csi"] = "cinder" labels["cinder-csi-version"] = csi_version + if utils.is_manila_csi_enabled(self.cluster): + manila_version = utils.get_cluster_label( + self.cluster, "manila_csi_plugin_tag", MANILA_CSI_TAG + ) + labels["manila-csi-version"] = manila_version + return {**super().labels, **labels} def get_or_none(self) -> objects.Cluster: diff --git a/magnum_cluster_api/utils.py b/magnum_cluster_api/utils.py index fae46002..6c068d7e 100644 --- a/magnum_cluster_api/utils.py +++ b/magnum_cluster_api/utils.py @@ -21,12 +21,15 @@ import yaml from magnum import objects as magnum_objects from magnum.common import context, exception, octavia +from magnum.common.keystone import KeystoneClientV3 from oslo_serialization import base64 from oslo_utils import strutils from tenacity import retry, retry_if_exception_type from magnum_cluster_api import clients, image_utils, images, objects +LOG = logging.getLogger(__name__) + def get_or_generate_cluster_api_cloud_config_secret_name( api: pykube.HTTPClient, cluster: magnum_objects.Cluster @@ -233,3 +236,56 @@ def delete_loadbalancers(ctx, cluster): octavia.wait_for_lb_deleted(octavia_client, candidates) except Exception as e: raise exception.PreDeletionFailed(cluster_uuid=cluster.uuid, msg=str(e)) + + +def is_cinder_enabled(): + """Check if Cinder service is deployed in the cloud.""" + + admin_context = context.make_admin_context() + keystone = KeystoneClientV3(admin_context) + + try: + cinder_svc = keystone.client.services.list(type="volumev3") + except Exception: + LOG.exception("Failed to list services") + raise exception.ServicesListFailed() + + # Always assume there is only one load balancing service configured. + if cinder_svc and cinder_svc[0].enabled: + return True + + LOG.info("There is no volumev3 service enabled in the cloud.") + return False + + +def is_manila_enabled(): + """Check if Manila service is deployed in the cloud.""" + + admin_context = context.make_admin_context() + keystone = KeystoneClientV3(admin_context) + + try: + manila_svc = keystone.client.services.list(type="sharev2") + except Exception: + LOG.exception("Failed to list services") + raise exception.ServicesListFailed() + + if manila_svc and manila_svc[0].enabled: + return True + + LOG.info("There is no sharev2 service enabled in the cloud.") + return False + + +def is_cinder_csi_enabled(cluster: magnum_objects.Cluster) -> bool: + return ( + get_cluster_label_as_bool(cluster, "cinder_csi_enabled", True) + and clients.is_cinder_enabled() + ) + + +def is_manila_csi_enabled(cluster: magnum_objects.Cluster) -> bool: + return ( + get_cluster_label_as_bool(cluster, "manila_csi_enabled", True) + and clients.is_manila_enabled() + ) From 7e97e18f519b3a761e632e9045a11015890c6afa Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Thu, 25 May 2023 02:19:28 +1000 Subject: [PATCH 02/20] Fix lint errors --- magnum_cluster_api/clients.py | 4 ---- magnum_cluster_api/utils.py | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/magnum_cluster_api/clients.py b/magnum_cluster_api/clients.py index b831d69f..c80bc471 100644 --- a/magnum_cluster_api/clients.py +++ b/magnum_cluster_api/clients.py @@ -53,9 +53,5 @@ def get_pykube_api() -> pykube.HTTPClient: return pykube.HTTPClient(pykube.KubeConfig.from_env()) -def get_openstack_api(context) -> clients.OpenStackClients: - return clients.OpenStackClients(context) - - def get_openstack_api(context) -> OpenStackClients: return OpenStackClients(context) diff --git a/magnum_cluster_api/utils.py b/magnum_cluster_api/utils.py index 6c068d7e..6182d85c 100644 --- a/magnum_cluster_api/utils.py +++ b/magnum_cluster_api/utils.py @@ -22,6 +22,7 @@ from magnum import objects as magnum_objects from magnum.common import context, exception, octavia from magnum.common.keystone import KeystoneClientV3 +from oslo_log import log as logging from oslo_serialization import base64 from oslo_utils import strutils from tenacity import retry, retry_if_exception_type From 42ec686cec0430283bdba2e79ec93e5c9f042ca7 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Thu, 25 May 2023 02:24:20 +1000 Subject: [PATCH 03/20] Fix registering common security config opts --- magnum_cluster_api/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magnum_cluster_api/conf.py b/magnum_cluster_api/conf.py index a37e18e1..e10b4c79 100644 --- a/magnum_cluster_api/conf.py +++ b/magnum_cluster_api/conf.py @@ -98,4 +98,4 @@ CONF.register_group(manila_client_group) CONF.register_opts(auto_scaling_opts, group=auto_scaling_group) CONF.register_opts(manila_client_opts, group=manila_client_group) -CONF.register_opts(manila_client_opts, group=common_security_opts) +CONF.register_opts(common_security_opts, group=manila_client_group) From 8e9b2bd2c0678125ad977d4a398a5c387144c8ab Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Thu, 25 May 2023 03:04:37 +1000 Subject: [PATCH 04/20] Add manilaclient in dep list --- poetry.lock | 28 +++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index e0a7958e..79c99f4c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2259,6 +2259,32 @@ requests = ">=2.14.2" six = ">=1.10.0" stevedore = ">=1.20.0" +[[package]] +name = "python-manilaclient" +version = "3.4.0" +description = "Client library for OpenStack Manila API." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "python-manilaclient-3.4.0.tar.gz", hash = "sha256:178d7f93b3498a0c05370dbde3ab23ddd6432831fe3e480e8e49a5f6ddcc82dc"}, + {file = "python_manilaclient-3.4.0-py3-none-any.whl", hash = "sha256:0ce99a63922d5c91a4380cc77135295bce8fbadaff3b01b694d21caa4698723d"}, +] + +[package.dependencies] +Babel = ">=2.3.4,<2.4.0 || >2.4.0" +debtcollector = ">=1.2.0" +osc-lib = ">=1.10.0" +"oslo.config" = ">=5.2.0" +"oslo.log" = ">=3.36.0" +"oslo.serialization" = ">=2.18.0,<2.19.1 || >2.19.1" +"oslo.utils" = ">=3.33.0" +pbr = ">=2.0.0,<2.1.0 || >2.1.0" +PrettyTable = ">=0.7.1" +python-keystoneclient = ">=3.8.0" +requests = ">=2.14.2" +simplejson = ">=3.5.1" + [[package]] name = "python-neutronclient" version = "7.8.0" @@ -3298,4 +3324,4 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=4.6)", "pytest-black ( [metadata] lock-version = "2.0" python-versions = "^3.6" -content-hash = "425ba2d0c191127787a20b8d902e6b5567bb293f57e2fbc2a18a5b9c15d6f87a" +content-hash = "2e2b7cc8a6a50bccb7e20ba3c2a511be96c259b6b2e861b1907f366ba796f7f4" diff --git a/pyproject.toml b/pyproject.toml index e3410f46..185704f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ magnum = ">=14.0.0" pykube-ng = "*" pyroute2 = ">=0.3.4" python = "^3.6" +"python-manilaclient" = ">=3.4.0" requests = ">=2.27.1" semver = "^2.0.0" shortuuid = "*" From 47320969bdcf08b1b4674677c4a8268c24962c80 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Thu, 25 May 2023 03:13:44 +1000 Subject: [PATCH 05/20] Downgrade the manilaclient version constraint --- poetry.lock | 20 +++++++------------- pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/poetry.lock b/poetry.lock index 79c99f4c..3d579fd5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1828,23 +1828,17 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "prettytable" -version = "2.5.0" -description = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format" +version = "0.7.2" +description = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format." category = "main" optional = false -python-versions = ">=3.6" +python-versions = "*" files = [ - {file = "prettytable-2.5.0-py3-none-any.whl", hash = "sha256:1411c65d21dca9eaa505ba1d041bed75a6d629ae22f5109a923f4e719cfecba4"}, - {file = "prettytable-2.5.0.tar.gz", hash = "sha256:f7da57ba63d55116d65e5acb147bfdfa60dceccabf0d607d6817ee2888a05f2c"}, + {file = "prettytable-0.7.2.tar.bz2", hash = "sha256:853c116513625c738dc3ce1aee148b5b5757a86727e67eff6502c7ca59d43c36"}, + {file = "prettytable-0.7.2.tar.gz", hash = "sha256:2d5460dc9db74a32bcc8f9f67de68b2c4f4d2f01fa3bd518764c69156d9cacd9"}, + {file = "prettytable-0.7.2.zip", hash = "sha256:a53da3b43d7a5c229b5e3ca2892ef982c46b7923b51e98f0db49956531211c4f"}, ] -[package.dependencies] -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} -wcwidth = "*" - -[package.extras] -tests = ["pytest", "pytest-cov", "pytest-lazy-fixture"] - [[package]] name = "prometheus-client" version = "0.16.0" @@ -3324,4 +3318,4 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=4.6)", "pytest-black ( [metadata] lock-version = "2.0" python-versions = "^3.6" -content-hash = "2e2b7cc8a6a50bccb7e20ba3c2a511be96c259b6b2e861b1907f366ba796f7f4" +content-hash = "28731eafe7a19ea37e086bc04e61a0f42cea95caabf23565f5e93e902598c9a1" diff --git a/pyproject.toml b/pyproject.toml index 185704f5..f2bed036 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ magnum = ">=14.0.0" pykube-ng = "*" pyroute2 = ">=0.3.4" python = "^3.6" -"python-manilaclient" = ">=3.4.0" +"python-manilaclient" = ">=3.3.2" requests = ">=2.27.1" semver = "^2.0.0" shortuuid = "*" From cfa7ae1d594e733b7bbb65d8a87c494927f4caf1 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Thu, 25 May 2023 19:21:31 +1000 Subject: [PATCH 06/20] Add manila install in hack and add script for manila csi manifest sync --- hack/stack.sh | 16 +++++ .../manila-csi/csi-controllerplugin.yaml | 1 - ...si-manifests => sync-cinder-csi-manifests} | 2 +- tools/sync-manila-csi-manifests | 64 +++++++++++++++++++ 4 files changed, 81 insertions(+), 2 deletions(-) rename tools/{sync-csi-manifests => sync-cinder-csi-manifests} (97%) mode change 100755 => 100644 create mode 100644 tools/sync-manila-csi-manifests diff --git a/hack/stack.sh b/hack/stack.sh index d175d271..4293d4e6 100755 --- a/hack/stack.sh +++ b/hack/stack.sh @@ -70,6 +70,22 @@ enable_plugin magnum https://opendev.org/openstack/magnum [cluster_template] kubernetes_allowed_network_drivers = calico kubernetes_default_network_driver = calico + +# Manila +LIBS_FROM_GIT=python-manilaclient +enable_plugin manila https://opendev.org/openstack/manila +enable_plugin manila-ui https://opendev.org/openstack/manila-ui +enable_plugin manila-tempest-plugin https://opendev.org/openstack/manila-tempest-plugin + +SHARE_DRIVER=manila.share.drivers.generic.GenericShareDriver +MANILA_ENABLED_BACKENDS=generic +MANILA_OPTGROUP_generic_driver_handles_share_servers=True +MANILA_OPTGROUP_generic_connect_share_server_to_tenant_network=True +MANILA_DEFAULT_SHARE_TYPE_EXTRA_SPECS='snapshot_support=True create_share_from_snapshot_support=True' +MANILA_CONFIGURE_DEFAULT_TYPES=True + +MANILA_SERVICE_IMAGE_ENABLED=True +MANILA_USE_SERVICE_INSTANCE_PASSWORD=True EOF # Start DevStack deployment diff --git a/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml b/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml index e875e055..8eab6e41 100644 --- a/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml +++ b/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml @@ -125,4 +125,3 @@ spec: hostPath: path: /var/lib/kubelet/pods type: Directory - diff --git a/tools/sync-csi-manifests b/tools/sync-cinder-csi-manifests old mode 100755 new mode 100644 similarity index 97% rename from tools/sync-csi-manifests rename to tools/sync-cinder-csi-manifests index ef8ee959..30819f24 --- a/tools/sync-csi-manifests +++ b/tools/sync-cinder-csi-manifests @@ -82,5 +82,5 @@ for manifest in MANIFESTS: else: docs.append(doc) - with open(f"magnum_cluster_api/manifests/csi/{manifest}", "w") as fd: + with open(f"magnum_cluster_api/manifests/cinder-csi/{manifest}", "w") as fd: yaml.dump_all(docs, fd, default_flow_style=False) diff --git a/tools/sync-manila-csi-manifests b/tools/sync-manila-csi-manifests new file mode 100644 index 00000000..e09cdd52 --- /dev/null +++ b/tools/sync-manila-csi-manifests @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2023 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This script is used to sync the CCM manifests from the CCM repo to the +# manifests folder. + +import requests +import yaml + +MANIFESTS = set( + [ + "csi-controllerplugin-rbac.yaml", + "csi-controllerplugin.yaml", + "csi-nodeplugin-rbac.yaml", + "csi-nodeplugin.yaml", + "csidriver.yaml", + ] +) + + +for manifest in MANIFESTS: + url = f"https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/manila-csi-plugin/{manifest}" # noqa E501 + + docs = [] + r = requests.get(url) + + for doc in yaml.safe_load_all(r.text): + if doc["kind"] == "StatefulSet": + doc["spec"]["template"]["spec"]["containers"][3]["env"][3][ + "value" + ] = "unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock" + doc["spec"]["template"]["spec"]["containers"][3]["env"][4][ + "value" + ] = "NFS" + + if doc["kind"] == "DaemonSet": + doc["spec"]["template"]["spec"]["containers"][1]["env"][3][ + "value" + ] = "unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock" + doc["spec"]["template"]["spec"]["containers"][1]["env"][4][ + "value" + ] = "NFS" + + if doc["kind"] == "List": + for item in doc["items"]: + docs.append(item) + else: + docs.append(doc) + + with open(f"magnum_cluster_api/manifests/manila-csi/{manifest}", "w") as fd: + yaml.dump_all(docs, fd, default_flow_style=False) From 71f57366bd96ad278c6d22be5fe1addb67451d07 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Thu, 25 May 2023 21:06:34 +1000 Subject: [PATCH 07/20] fix order in stack.sh --- hack/stack.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hack/stack.sh b/hack/stack.sh index 4293d4e6..6fe09f31 100755 --- a/hack/stack.sh +++ b/hack/stack.sh @@ -66,11 +66,6 @@ enable_service octavia o-api o-cw o-hm o-hk o-da # Magnum enable_plugin magnum https://opendev.org/openstack/magnum -[[post-config|/etc/magnum/magnum.conf]] -[cluster_template] -kubernetes_allowed_network_drivers = calico -kubernetes_default_network_driver = calico - # Manila LIBS_FROM_GIT=python-manilaclient enable_plugin manila https://opendev.org/openstack/manila @@ -86,6 +81,11 @@ MANILA_CONFIGURE_DEFAULT_TYPES=True MANILA_SERVICE_IMAGE_ENABLED=True MANILA_USE_SERVICE_INSTANCE_PASSWORD=True + +[[post-config|/etc/magnum/magnum.conf]] +[cluster_template] +kubernetes_allowed_network_drivers = calico +kubernetes_default_network_driver = calico EOF # Start DevStack deployment From b8bdc62ca2e88a71c6810af8fa28a6fd0cb631eb Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Fri, 26 May 2023 23:10:47 +1000 Subject: [PATCH 08/20] Fix image override and set manilaclient api version --- magnum_cluster_api/clients.py | 5 +---- magnum_cluster_api/image_utils.py | 2 ++ magnum_cluster_api/utils.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/magnum_cluster_api/clients.py b/magnum_cluster_api/clients.py index c80bc471..c91ec4d7 100644 --- a/magnum_cluster_api/clients.py +++ b/magnum_cluster_api/clients.py @@ -41,10 +41,7 @@ def manila(self): session = self.keystone().session self._manila = manilaclient.Client( - api_version=manilaclient_version, - session=session, - service_catalog_url=endpoint, - **args + manilaclient_version, session=session, service_catalog_url=endpoint, **args ) return self._manila diff --git a/magnum_cluster_api/image_utils.py b/magnum_cluster_api/image_utils.py index efc795ea..aa60b289 100644 --- a/magnum_cluster_api/image_utils.py +++ b/magnum_cluster_api/image_utils.py @@ -63,6 +63,8 @@ def get_image(name: str, repository: str = None): new_image_name = name.replace("docker.io/k8scloudprovider", repository) if name.startswith("registry.k8s.io/sig-storage"): new_image_name = name.replace("registry.k8s.io/sig-storage", repository) + if name.startswith("registry.k8s.io/provider-os"): + new_image_name = name.replace("registry.k8s.io/provider-os", repository) if new_image_name.startswith(f"{repository}/livenessprobe"): return new_image_name.replace("livenessprobe", "csi-livenessprobe") if new_image_name.startswith("registry.k8s.io/coredns"): diff --git a/magnum_cluster_api/utils.py b/magnum_cluster_api/utils.py index 6182d85c..2581bfcb 100644 --- a/magnum_cluster_api/utils.py +++ b/magnum_cluster_api/utils.py @@ -281,12 +281,12 @@ def is_manila_enabled(): def is_cinder_csi_enabled(cluster: magnum_objects.Cluster) -> bool: return ( get_cluster_label_as_bool(cluster, "cinder_csi_enabled", True) - and clients.is_cinder_enabled() + and is_cinder_enabled() ) def is_manila_csi_enabled(cluster: magnum_objects.Cluster) -> bool: return ( get_cluster_label_as_bool(cluster, "manila_csi_enabled", True) - and clients.is_manila_enabled() + and is_manila_enabled() ) From 2adf3b5df3a97e03b208e0448245b9e94f8044d2 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Sat, 27 May 2023 00:14:17 +1000 Subject: [PATCH 09/20] Update sync script for manila csi - set the namespace in namespace-scoped resources' metadata - set the namespace in subjets of role binding resources --- .../manila-csi/csi-controllerplugin-rbac.yaml | 219 ++++++++++++------ .../manila-csi/csi-controllerplugin.yaml | 190 ++++++++------- .../manila-csi/csi-nodeplugin-rbac.yaml | 78 ++++--- .../manifests/manila-csi/csi-nodeplugin.yaml | 153 ++++++------ .../manifests/manila-csi/csidriver.yaml | 1 - tools/sync-manila-csi-manifests | 30 ++- 6 files changed, 389 insertions(+), 282 deletions(-) diff --git a/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin-rbac.yaml b/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin-rbac.yaml index cbf3c601..968e7a59 100644 --- a/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin-rbac.yaml +++ b/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin-rbac.yaml @@ -1,113 +1,196 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: openstack-manila-csi-controllerplugin labels: app: openstack-manila-csi component: controllerplugin + name: openstack-manila-csi-controllerplugin + namespace: kube-system --- -kind: ClusterRole +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-controllerplugin: 'true' apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: - name: openstack-manila-csi-controllerplugin labels: app: openstack-manila-csi component: controllerplugin -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-controllerplugin: "true" + name: openstack-manila-csi-controllerplugin rules: [] --- -kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: - name: openstack-manila-csi-controllerplugin-rules labels: app: openstack-manila-csi component: controllerplugin - rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-controllerplugin: "true" + rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-controllerplugin: 'true' + name: openstack-manila-csi-controllerplugin-rules rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update", "patch"] +- apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - '' + resources: + - secrets + verbs: + - get + - list +- apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - patch +- apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update +- apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - patch +- apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch --- -kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: - name: openstack-manila-csi-controllerplugin labels: app: openstack-manila-csi component: controllerplugin -subjects: - - kind: ServiceAccount - name: openstack-manila-csi-controllerplugin - namespace: default + name: openstack-manila-csi-controllerplugin roleRef: + apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: openstack-manila-csi-controllerplugin - apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: openstack-manila-csi-controllerplugin + namespace: kube-system --- -kind: Role apiVersion: rbac.authorization.k8s.io/v1 +kind: Role metadata: - name: openstack-manila-csi-controllerplugin labels: app: openstack-manila-csi component: controllerplugin + name: openstack-manila-csi-controllerplugin + namespace: kube-system rules: - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "delete"] +- apiGroups: + - '' + resources: + - endpoints + verbs: + - get + - watch + - list + - delete + - update + - create +- apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - delete --- -kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding metadata: - name: openstack-manila-csi-controllerplugin labels: app: openstack-manila-csi component: controllerplugin -subjects: - - kind: ServiceAccount - name: openstack-manila-csi-controllerplugin - namespace: default + name: openstack-manila-csi-controllerplugin + namespace: kube-system roleRef: + apiGroup: rbac.authorization.k8s.io kind: Role name: openstack-manila-csi-controllerplugin - apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: openstack-manila-csi-controllerplugin + namespace: kube-system diff --git a/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml b/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml index 8eab6e41..6c1d1155 100644 --- a/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml +++ b/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml @@ -1,127 +1,121 @@ -kind: Service apiVersion: v1 +kind: Service metadata: - name: openstack-manila-csi-controllerplugin labels: app: openstack-manila-csi component: controllerplugin + name: openstack-manila-csi-controllerplugin + namespace: kube-system spec: + ports: + - name: dummy + port: 12345 selector: app: openstack-manila-csi component: controllerplugin - ports: - - name: dummy - port: 12345 --- -kind: StatefulSet apiVersion: apps/v1 +kind: StatefulSet metadata: - name: openstack-manila-csi-controllerplugin labels: app: openstack-manila-csi component: controllerplugin + name: openstack-manila-csi-controllerplugin + namespace: kube-system spec: - serviceName: openstack-manila-csi-controllerplugin replicas: 1 selector: matchLabels: app: openstack-manila-csi component: controllerplugin + serviceName: openstack-manila-csi-controllerplugin template: metadata: labels: app: openstack-manila-csi component: controllerplugin spec: - serviceAccountName: openstack-manila-csi-controllerplugin containers: - - name: provisioner - image: "registry.k8s.io/sig-storage/csi-provisioner:v3.0.0" - args: - - "--csi-address=$(ADDRESS)" - # To enable topology awareness in csi-provisioner, uncomment the following line: - # - "--feature-gates=Topology=true" - env: - - name: ADDRESS - value: "unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi-controllerplugin.sock" - imagePullPolicy: IfNotPresent - volumeMounts: - - name: plugin-dir - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org - - name: snapshotter - image: "registry.k8s.io/sig-storage/csi-snapshotter:v5.0.1" - args: - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: "unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi-controllerplugin.sock" - imagePullPolicy: IfNotPresent - volumeMounts: - - name: plugin-dir - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org - - name: resizer - image: "registry.k8s.io/sig-storage/csi-resizer:v1.3.0" - args: - - "--csi-address=$(ADDRESS)" - - "--handle-volume-inuse-error=false" - env: - - name: ADDRESS - value: "unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi-controllerplugin.sock" - imagePullPolicy: IfNotPresent - volumeMounts: - - name: plugin-dir - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org - - name: nodeplugin - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - image: registry.k8s.io/provider-os/manila-csi-plugin:v1.27.0-alpha.0 - command: ["/bin/sh", "-c", - '/bin/manila-csi-plugin - --nodeid=$(NODE_ID) - --endpoint=$(CSI_ENDPOINT) - --drivername=$(DRIVER_NAME) - --share-protocol-selector=$(MANILA_SHARE_PROTO) - --fwdendpoint=$(FWD_CSI_ENDPOINT)' - # To enable topology awareness and retrieve compute node AZs from the OpenStack Metadata Service, add the following flags: - # --with-topology - # --nodeaz=$(curl http://169.254.169.254/openstack/latest/meta_data.json | jq -r .availability_zone) - # Those flags need to be added to csi-nodeplugin.yaml as well. - ] - env: - - name: DRIVER_NAME - value: manila.csi.openstack.org - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: "unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi-controllerplugin.sock" - - name: FWD_CSI_ENDPOINT - value: "unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock" - - name: MANILA_SHARE_PROTO - value: "NFS" - imagePullPolicy: IfNotPresent - volumeMounts: - - name: plugin-dir - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org - - name: fwd-plugin-dir - mountPath: /var/lib/kubelet/plugins/csi-nfsplugin - - name: pod-mounts - mountPath: /var/lib/kubelet/pods - mountPropagation: Bidirectional + - args: + - --csi-address=$(ADDRESS) + env: + - name: ADDRESS + value: unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi-controllerplugin.sock + image: registry.k8s.io/sig-storage/csi-provisioner:v3.0.0 + imagePullPolicy: IfNotPresent + name: provisioner + volumeMounts: + - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org + name: plugin-dir + - args: + - --csi-address=$(ADDRESS) + env: + - name: ADDRESS + value: unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi-controllerplugin.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v5.0.1 + imagePullPolicy: IfNotPresent + name: snapshotter + volumeMounts: + - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org + name: plugin-dir + - args: + - --csi-address=$(ADDRESS) + - --handle-volume-inuse-error=false + env: + - name: ADDRESS + value: unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi-controllerplugin.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + imagePullPolicy: IfNotPresent + name: resizer + volumeMounts: + - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org + name: plugin-dir + - command: + - /bin/sh + - -c + - /bin/manila-csi-plugin --nodeid=$(NODE_ID) --endpoint=$(CSI_ENDPOINT) --drivername=$(DRIVER_NAME) + --share-protocol-selector=$(MANILA_SHARE_PROTO) --fwdendpoint=$(FWD_CSI_ENDPOINT) + env: + - name: DRIVER_NAME + value: manila.csi.openstack.org + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi-controllerplugin.sock + - name: FWD_CSI_ENDPOINT + value: unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock + - name: MANILA_SHARE_PROTO + value: NFS + image: registry.k8s.io/provider-os/manila-csi-plugin:v1.27.1 + imagePullPolicy: IfNotPresent + name: nodeplugin + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org + name: plugin-dir + - mountPath: /var/lib/kubelet/plugins/FWD-NODEPLUGIN + name: fwd-plugin-dir + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: pod-mounts + serviceAccountName: openstack-manila-csi-controllerplugin volumes: - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/manila.csi.openstack.org - type: DirectoryOrCreate - - name: fwd-plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/csi-nfsplugin - type: Directory - - name: pod-mounts - hostPath: - path: /var/lib/kubelet/pods - type: Directory + - hostPath: + path: /var/lib/kubelet/plugins/manila.csi.openstack.org + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins/FWD-NODEPLUGIN + type: Directory + name: fwd-plugin-dir + - hostPath: + path: /var/lib/kubelet/pods + type: Directory + name: pod-mounts diff --git a/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin-rbac.yaml b/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin-rbac.yaml index 6d6228cd..f946e6a3 100644 --- a/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin-rbac.yaml +++ b/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin-rbac.yaml @@ -1,58 +1,78 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: openstack-manila-csi-nodeplugin labels: app: openstack-manila-csi component: nodeplugin + name: openstack-manila-csi-nodeplugin + namespace: kube-system --- -kind: ClusterRole +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-nodeplugin: 'true' apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: - name: openstack-manila-csi-nodeplugin labels: app: openstack-manila-csi component: nodeplugin -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-nodeplugin: "true" + name: openstack-manila-csi-nodeplugin rules: [] --- -kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: - name: openstack-manila-csi-nodeplugin-rules labels: app: openstack-manila-csi component: nodeplugin - rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-nodeplugin: "true" + rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-nodeplugin: 'true' + name: openstack-manila-csi-nodeplugin-rules rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] +- apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - list +- apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - update +- apiGroups: + - '' + resources: + - namespaces + verbs: + - get + - list +- apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update --- -kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: - name: openstack-manila-csi-nodeplugin labels: app: openstack-manila-csi component: nodeplugin -subjects: - - kind: ServiceAccount - name: openstack-manila-csi-nodeplugin - namespace: default + name: openstack-manila-csi-nodeplugin roleRef: + apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: openstack-manila-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: openstack-manila-csi-nodeplugin + namespace: kube-system diff --git a/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml b/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml index 8bf4b3c7..5934fe68 100644 --- a/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml +++ b/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml @@ -1,10 +1,11 @@ -kind: DaemonSet apiVersion: apps/v1 +kind: DaemonSet metadata: - name: openstack-manila-csi-nodeplugin labels: app: openstack-manila-csi component: nodeplugin + name: openstack-manila-csi-nodeplugin + namespace: kube-system spec: selector: matchLabels: @@ -16,83 +17,75 @@ spec: app: openstack-manila-csi component: nodeplugin spec: - serviceAccountName: openstack-manila-csi-nodeplugin - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet containers: - - name: registrar - image: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.4.0" - args: - - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/manila.csi.openstack.org/csi.sock" - lifecycle: - preStop: - exec: - command: [ - "/bin/sh", "-c", - 'rm -rf /registration/manila.csi.openstack.org - /registration/manila.csi.openstack.org-reg.sock' - ] - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - imagePullPolicy: IfNotPresent - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - - name: nodeplugin - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - image: registry.k8s.io/provider-os/manila-csi-plugin:latest - command: ["/bin/sh", "-c", - '/bin/manila-csi-plugin - --nodeid=$(NODE_ID) - --endpoint=$(CSI_ENDPOINT) - --drivername=$(DRIVER_NAME) - --share-protocol-selector=$(MANILA_SHARE_PROTO) - --fwdendpoint=$(FWD_CSI_ENDPOINT)' - # To enable topology awareness and retrieve compute node AZs from the OpenStack Metadata Service, add the following flags: - # --with-topology - # --nodeaz=$(curl http://169.254.169.254/openstack/latest/meta_data.json | jq -r .availability_zone) - # Those flags need to be added to csi-controllerplugin.yaml as well. - ] - env: - - name: DRIVER_NAME - value: manila.csi.openstack.org - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: "unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi.sock" - - name: FWD_CSI_ENDPOINT - value: "unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock" - - name: MANILA_SHARE_PROTO - value: "NFS" - imagePullPolicy: IfNotPresent - volumeMounts: - - name: plugin-dir - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org - - name: fwd-plugin-dir - mountPath: /var/lib/kubelet/plugins/csi-nfsplugin + - args: + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/manila.csi.openstack.org/csi.sock + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.4.0 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - rm -rf /registration/manila.csi.openstack.org /registration/manila.csi.openstack.org-reg.sock + name: registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - command: + - /bin/sh + - -c + - /bin/manila-csi-plugin --nodeid=$(NODE_ID) --endpoint=$(CSI_ENDPOINT) --drivername=$(DRIVER_NAME) + --share-protocol-selector=$(MANILA_SHARE_PROTO) --fwdendpoint=$(FWD_CSI_ENDPOINT) + env: + - name: DRIVER_NAME + value: manila.csi.openstack.org + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi.sock + - name: FWD_CSI_ENDPOINT + value: unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock + - name: MANILA_SHARE_PROTO + value: NFS + image: registry.k8s.io/provider-os/manila-csi-plugin:v1.27.1 + imagePullPolicy: IfNotPresent + name: nodeplugin + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org + name: plugin-dir + - mountPath: /var/lib/kubelet/plugins/FWD-NODEPLUGIN + name: fwd-plugin-dir + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + serviceAccountName: openstack-manila-csi-nodeplugin volumes: - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry - type: Directory - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/manila.csi.openstack.org - type: DirectoryOrCreate - - name: fwd-plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/csi-nfsplugin - type: DirectoryOrCreate - + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins/manila.csi.openstack.org + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins/FWD-NODEPLUGIN + type: DirectoryOrCreate + name: fwd-plugin-dir diff --git a/magnum_cluster_api/manifests/manila-csi/csidriver.yaml b/magnum_cluster_api/manifests/manila-csi/csidriver.yaml index 5d95ecd8..6bbbf48d 100644 --- a/magnum_cluster_api/manifests/manila-csi/csidriver.yaml +++ b/magnum_cluster_api/manifests/manila-csi/csidriver.yaml @@ -5,4 +5,3 @@ metadata: spec: attachRequired: false podInfoOnMount: false - diff --git a/tools/sync-manila-csi-manifests b/tools/sync-manila-csi-manifests index e09cdd52..9d2a2a6c 100644 --- a/tools/sync-manila-csi-manifests +++ b/tools/sync-manila-csi-manifests @@ -31,6 +31,26 @@ MANIFESTS = set( ) +def set_namespace(doc, namespace="kube-system"): + cluster_scope_kinds = [ + "ClusterRole", + "ClusterRoleBinding", + "CSIDriver", + "List", + ] + binding_kinds = [ + "ClusterRoleBinding", + "RoleBinding", + ] + if doc["kind"] not in cluster_scope_kinds: + doc["metadata"]["namespace"] = namespace + + if doc["kind"] in binding_kinds: + for item in doc["subjects"]: + item["namespace"] = namespace + return doc + + for manifest in MANIFESTS: url = f"https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/manila-csi-plugin/{manifest}" # noqa E501 @@ -42,22 +62,20 @@ for manifest in MANIFESTS: doc["spec"]["template"]["spec"]["containers"][3]["env"][3][ "value" ] = "unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock" - doc["spec"]["template"]["spec"]["containers"][3]["env"][4][ - "value" - ] = "NFS" + doc["spec"]["template"]["spec"]["containers"][3]["env"][4]["value"] = "NFS" if doc["kind"] == "DaemonSet": doc["spec"]["template"]["spec"]["containers"][1]["env"][3][ "value" ] = "unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock" - doc["spec"]["template"]["spec"]["containers"][1]["env"][4][ - "value" - ] = "NFS" + doc["spec"]["template"]["spec"]["containers"][1]["env"][4]["value"] = "NFS" if doc["kind"] == "List": for item in doc["items"]: + item["metadata"]["namespace"] = "kube-system" docs.append(item) else: + doc = set_namespace(doc) docs.append(doc) with open(f"magnum_cluster_api/manifests/manila-csi/{manifest}", "w") as fd: From 76280d6e9c3982a4d67036241551b379b732fbc6 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Sat, 27 May 2023 00:53:41 +1000 Subject: [PATCH 10/20] Set the image tag as latest --- .../manifests/manila-csi/csi-controllerplugin.yaml | 2 +- magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml | 2 +- tools/sync-manila-csi-manifests | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml b/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml index 6c1d1155..45fb101e 100644 --- a/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml +++ b/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml @@ -88,7 +88,7 @@ spec: value: unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock - name: MANILA_SHARE_PROTO value: NFS - image: registry.k8s.io/provider-os/manila-csi-plugin:v1.27.1 + image: registry.k8s.io/provider-os/manila-csi-plugin:latest imagePullPolicy: IfNotPresent name: nodeplugin securityContext: diff --git a/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml b/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml index 5934fe68..b972f023 100644 --- a/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml +++ b/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml @@ -59,7 +59,7 @@ spec: value: unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock - name: MANILA_SHARE_PROTO value: NFS - image: registry.k8s.io/provider-os/manila-csi-plugin:v1.27.1 + image: registry.k8s.io/provider-os/manila-csi-plugin:latest imagePullPolicy: IfNotPresent name: nodeplugin securityContext: diff --git a/tools/sync-manila-csi-manifests b/tools/sync-manila-csi-manifests index 9d2a2a6c..01c76ac4 100644 --- a/tools/sync-manila-csi-manifests +++ b/tools/sync-manila-csi-manifests @@ -63,12 +63,14 @@ for manifest in MANIFESTS: "value" ] = "unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock" doc["spec"]["template"]["spec"]["containers"][3]["env"][4]["value"] = "NFS" + doc["spec"]["template"]["spec"]["containers"][3]["image"] = "registry.k8s.io/provider-os/manila-csi-plugin:latest" if doc["kind"] == "DaemonSet": doc["spec"]["template"]["spec"]["containers"][1]["env"][3][ "value" ] = "unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock" doc["spec"]["template"]["spec"]["containers"][1]["env"][4]["value"] = "NFS" + doc["spec"]["template"]["spec"]["containers"][1]["image"] = "registry.k8s.io/provider-os/manila-csi-plugin:latest" if doc["kind"] == "List": for item in doc["items"]: From 883c85fc1af77111cac3ab5e86e4256fc34a59c1 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Sat, 27 May 2023 02:12:43 +1000 Subject: [PATCH 11/20] Add nfs-csi and set tolerations for manila-csi nodeplugin --- .../manila-csi/csi-controllerplugin.yaml | 4 +- .../manifests/manila-csi/csi-nodeplugin.yaml | 6 +- .../manifests/nfs-csi/csi-nfs-controller.yaml | 136 ++++++++++++++++++ .../manifests/nfs-csi/csi-nfs-driverinfo.yaml | 9 ++ .../manifests/nfs-csi/csi-nfs-node.yaml | 135 +++++++++++++++++ .../manifests/nfs-csi/rbac-csi-nfs.yaml | 128 +++++++++++++++++ tools/sync-manila-csi-manifests | 14 +- tools/sync-nfs-csi-manifests | 47 ++++++ 8 files changed, 471 insertions(+), 8 deletions(-) create mode 100644 magnum_cluster_api/manifests/nfs-csi/csi-nfs-controller.yaml create mode 100644 magnum_cluster_api/manifests/nfs-csi/csi-nfs-driverinfo.yaml create mode 100644 magnum_cluster_api/manifests/nfs-csi/csi-nfs-node.yaml create mode 100644 magnum_cluster_api/manifests/nfs-csi/rbac-csi-nfs.yaml create mode 100644 tools/sync-nfs-csi-manifests diff --git a/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml b/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml index 45fb101e..338ec7ba 100644 --- a/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml +++ b/magnum_cluster_api/manifests/manila-csi/csi-controllerplugin.yaml @@ -100,7 +100,7 @@ spec: volumeMounts: - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org name: plugin-dir - - mountPath: /var/lib/kubelet/plugins/FWD-NODEPLUGIN + - mountPath: /var/lib/kubelet/plugins/csi-nfsplugin name: fwd-plugin-dir - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional @@ -112,7 +112,7 @@ spec: type: DirectoryOrCreate name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/FWD-NODEPLUGIN + path: /var/lib/kubelet/plugins/csi-nfsplugin type: Directory name: fwd-plugin-dir - hostPath: diff --git a/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml b/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml index b972f023..cbe0a532 100644 --- a/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml +++ b/magnum_cluster_api/manifests/manila-csi/csi-nodeplugin.yaml @@ -71,11 +71,13 @@ spec: volumeMounts: - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org name: plugin-dir - - mountPath: /var/lib/kubelet/plugins/FWD-NODEPLUGIN + - mountPath: /var/lib/kubelet/plugins/csi-nfsplugin name: fwd-plugin-dir dnsPolicy: ClusterFirstWithHostNet hostNetwork: true serviceAccountName: openstack-manila-csi-nodeplugin + tolerations: + - operator: Exists volumes: - hostPath: path: /var/lib/kubelet/plugins_registry @@ -86,6 +88,6 @@ spec: type: DirectoryOrCreate name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/FWD-NODEPLUGIN + path: /var/lib/kubelet/plugins/csi-nfsplugin type: DirectoryOrCreate name: fwd-plugin-dir diff --git a/magnum_cluster_api/manifests/nfs-csi/csi-nfs-controller.yaml b/magnum_cluster_api/manifests/nfs-csi/csi-nfs-controller.yaml new file mode 100644 index 00000000..859768a3 --- /dev/null +++ b/magnum_cluster_api/manifests/nfs-csi/csi-nfs-controller.yaml @@ -0,0 +1,136 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: csi-nfs-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: csi-nfs-controller + template: + metadata: + labels: + app: csi-nfs-controller + spec: + containers: + - args: + - -v=2 + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-namespace=kube-system + - --extra-create-metadata=true + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 + name: csi-provisioner + resources: + limits: + memory: 400Mi + requests: + cpu: 10m + memory: 20Mi + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --v=5 + - --csi-address=$(ADDRESS) + - --leader-election-namespace=kube-system + - --leader-election + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1 + imagePullPolicy: IfNotPresent + name: csi-snapshotter + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=29652 + - --v=2 + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + name: liveness-probe + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - -v=5 + - --nodeid=$(NODE_ID) + - --endpoint=$(CSI_ENDPOINT) + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + image: gcr.io/k8s-staging-sig-storage/nfsplugin:canary + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + name: nfs + ports: + - containerPort: 29652 + name: healthz + protocol: TCP + resources: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /csi + name: socket-dir + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + seccompProfile: + type: RuntimeDefault + serviceAccountName: csi-nfs-controller-sa + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/controlplane + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet/pods + type: Directory + name: pods-mount-dir + - emptyDir: {} + name: socket-dir diff --git a/magnum_cluster_api/manifests/nfs-csi/csi-nfs-driverinfo.yaml b/magnum_cluster_api/manifests/nfs-csi/csi-nfs-driverinfo.yaml new file mode 100644 index 00000000..fc933360 --- /dev/null +++ b/magnum_cluster_api/manifests/nfs-csi/csi-nfs-driverinfo.yaml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: nfs.csi.k8s.io +spec: + attachRequired: false + fsGroupPolicy: File + volumeLifecycleModes: + - Persistent diff --git a/magnum_cluster_api/manifests/nfs-csi/csi-nfs-node.yaml b/magnum_cluster_api/manifests/nfs-csi/csi-nfs-node.yaml new file mode 100644 index 00000000..edaf92cc --- /dev/null +++ b/magnum_cluster_api/manifests/nfs-csi/csi-nfs-node.yaml @@ -0,0 +1,135 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: csi-nfs-node + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-nfs-node + template: + metadata: + labels: + app: csi-nfs-node + spec: + containers: + - args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=29653 + - --v=2 + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + name: liveness-probe + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --v=2 + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + env: + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi-nfsplugin/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + name: node-driver-registrar + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /registration + name: registration-dir + - args: + - -v=5 + - --nodeid=$(NODE_ID) + - --endpoint=$(CSI_ENDPOINT) + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + image: gcr.io/k8s-staging-sig-storage/nfsplugin:canary + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + name: nfs + ports: + - containerPort: 29653 + name: healthz + protocol: TCP + resources: + limits: + memory: 300Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: pods-mount-dir + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault + serviceAccountName: csi-nfs-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-nfsplugin + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: /var/lib/kubelet/pods + type: Directory + name: pods-mount-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate diff --git a/magnum_cluster_api/manifests/nfs-csi/rbac-csi-nfs.yaml b/magnum_cluster_api/manifests/nfs-csi/rbac-csi-nfs.yaml new file mode 100644 index 00000000..1ad3810e --- /dev/null +++ b/magnum_cluster_api/manifests/nfs-csi/rbac-csi-nfs.yaml @@ -0,0 +1,128 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-nfs-controller-sa + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-nfs-node-sa + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nfs-external-provisioner-role +rules: +- apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + - volumesnapshots + verbs: + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - get + - update + - patch +- apiGroups: + - '' + resources: + - events + verbs: + - get + - list + - watch + - create + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch +- apiGroups: + - '' + resources: + - secrets + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nfs-csi-provisioner-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nfs-external-provisioner-role +subjects: +- kind: ServiceAccount + name: csi-nfs-controller-sa + namespace: kube-system diff --git a/tools/sync-manila-csi-manifests b/tools/sync-manila-csi-manifests index 01c76ac4..d3b7f56c 100644 --- a/tools/sync-manila-csi-manifests +++ b/tools/sync-manila-csi-manifests @@ -64,6 +64,9 @@ for manifest in MANIFESTS: ] = "unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock" doc["spec"]["template"]["spec"]["containers"][3]["env"][4]["value"] = "NFS" doc["spec"]["template"]["spec"]["containers"][3]["image"] = "registry.k8s.io/provider-os/manila-csi-plugin:latest" + doc["spec"]["template"]["spec"]["containers"][3]["volumeMounts"][1]["mountPath"] = "/var/lib/kubelet/plugins/csi-nfsplugin" + doc["spec"]["template"]["spec"]["volumes"][1]["hostPath"]["path"] = "/var/lib/kubelet/plugins/csi-nfsplugin" + if doc["kind"] == "DaemonSet": doc["spec"]["template"]["spec"]["containers"][1]["env"][3][ @@ -71,14 +74,17 @@ for manifest in MANIFESTS: ] = "unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock" doc["spec"]["template"]["spec"]["containers"][1]["env"][4]["value"] = "NFS" doc["spec"]["template"]["spec"]["containers"][1]["image"] = "registry.k8s.io/provider-os/manila-csi-plugin:latest" + # Set FWD plugin + doc["spec"]["template"]["spec"]["containers"][1]["volumeMounts"][1]["mountPath"] = "/var/lib/kubelet/plugins/csi-nfsplugin" + doc["spec"]["template"]["spec"]["volumes"][2]["hostPath"]["path"] = "/var/lib/kubelet/plugins/csi-nfsplugin" + # Set toleration + doc["spec"]["template"]["spec"]["tolerations"] = [{"operator": "Exists"}] if doc["kind"] == "List": for item in doc["items"]: - item["metadata"]["namespace"] = "kube-system" - docs.append(item) + docs.append(set_namespace(item)) else: - doc = set_namespace(doc) - docs.append(doc) + docs.append(set_namespace(doc)) with open(f"magnum_cluster_api/manifests/manila-csi/{manifest}", "w") as fd: yaml.dump_all(docs, fd, default_flow_style=False) diff --git a/tools/sync-nfs-csi-manifests b/tools/sync-nfs-csi-manifests new file mode 100644 index 00000000..2c152f6e --- /dev/null +++ b/tools/sync-nfs-csi-manifests @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2023 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This script is used to sync the CCM manifests from the CCM repo to the +# manifests folder. + +import requests +import yaml + +MANIFESTS = set( + [ + "csi-nfs-node.yaml", + "csi-nfs-controller.yaml", + "csi-nfs-driverinfo.yaml", + "rbac-csi-nfs.yaml", + ] +) + + +for manifest in MANIFESTS: + url = f"https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/deploy/{manifest}" # noqa E501 + + docs = [] + r = requests.get(url) + + for doc in yaml.safe_load_all(r.text): + if doc["kind"] == "List": + for item in doc["items"]: + docs.append(item) + else: + docs.append(doc) + + with open(f"magnum_cluster_api/manifests/nfs-csi/{manifest}", "w") as fd: + yaml.dump_all(docs, fd, default_flow_style=False) From 9d98a4e1b6ef4cf815aa036e545fe72a4a520ec4 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Sat, 27 May 2023 02:18:36 +1000 Subject: [PATCH 12/20] Install nfs-csi as well as manila-csi --- magnum_cluster_api/resources.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index 3d1fdeaf..35265c73 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -37,6 +37,7 @@ CALICO_TAG = "v3.24.2" CINDER_CSI_TAG = "v1.25.3" MANILA_CSI_TAG = "v1.25.3" +NFS_CSI_TAG = "v4.2.0" CLUSTER_CLASS_VERSION = pkg_resources.require("magnum_cluster_api")[0].version CLUSTER_CLASS_NAME = f"magnum-v{CLUSTER_CLASS_VERSION}" @@ -251,6 +252,22 @@ def get_object(self) -> pykube.ConfigMap: ) data = { **data, + **{ + os.path.basename(manifest): image_utils.update_manifest_images( + self.cluster.uuid, + manifest, + repository=repository, + replacements=[ + ( + "gcr.io/k8s-staging-sig-storage/nfsplugin:canary", + f"gcr.io/k8s-staging-sig-storage/nfsplugin:{NFS_CSI_TAG}", + ), + ], + ) + for manifest in glob.glob( + os.path.join(manifests_path, "nfs-csi/*.yaml") + ) + }, **{ os.path.basename(manifest): image_utils.update_manifest_images( self.cluster.uuid, From aa67eda7a25d077abfbe0fa46eac6b0c6e54bc7c Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Sat, 27 May 2023 02:25:53 +1000 Subject: [PATCH 13/20] Add a condition for gcr.io/k8s-staging-sig-storage in image_utils --- magnum_cluster_api/image_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/magnum_cluster_api/image_utils.py b/magnum_cluster_api/image_utils.py index aa60b289..059d74e7 100644 --- a/magnum_cluster_api/image_utils.py +++ b/magnum_cluster_api/image_utils.py @@ -65,6 +65,8 @@ def get_image(name: str, repository: str = None): new_image_name = name.replace("registry.k8s.io/sig-storage", repository) if name.startswith("registry.k8s.io/provider-os"): new_image_name = name.replace("registry.k8s.io/provider-os", repository) + if name.startswith("gcr.io/k8s-staging-sig-storage"): + new_image_name = name.replace("gcr.io/k8s-staging-sig-storage", repository) if new_image_name.startswith(f"{repository}/livenessprobe"): return new_image_name.replace("livenessprobe", "csi-livenessprobe") if new_image_name.startswith("registry.k8s.io/coredns"): From aeb52e2915c2f2063c08d1d117866963cd8ee304 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Sat, 27 May 2023 03:42:25 +1000 Subject: [PATCH 14/20] Add images in image loader --- magnum_cluster_api/cmd/image_loader.py | 10 ++++++++++ magnum_cluster_api/image_utils.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/magnum_cluster_api/cmd/image_loader.py b/magnum_cluster_api/cmd/image_loader.py index e9f8c4ae..6c206181 100644 --- a/magnum_cluster_api/cmd/image_loader.py +++ b/magnum_cluster_api/cmd/image_loader.py @@ -27,7 +27,9 @@ "docker.io/calico/kube-controllers:v3.24.2", "docker.io/calico/node:v3.24.2", "docker.io/k8scloudprovider/cinder-csi-plugin:v1.25.3", + "docker.io/k8scloudprovider/manila-csi-plugin:v1.25.3", "docker.io/k8scloudprovider/openstack-cloud-controller-manager:v1.25.3", + "gcr.io/k8s-staging-sig-storage/nfsplugin:v4.2.0", "registry.k8s.io/coredns/coredns:v1.8.6", "registry.k8s.io/coredns/coredns:v1.9.3", CONF.auto_scaling.v1_22_image, @@ -62,11 +64,19 @@ "registry.k8s.io/kube-scheduler:v1.26.2", images.PAUSE, "registry.k8s.io/sig-storage/csi-attacher:v3.4.0", + "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.4.0", "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1", + "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0", + "registry.k8s.io/sig-storage/csi-provisioner:v3.0.0", "registry.k8s.io/sig-storage/csi-provisioner:v3.1.0", + "registry.k8s.io/sig-storage/csi-provisioner:v3.5.0", "registry.k8s.io/sig-storage/csi-resizer:v1.4.0", + "registry.k8s.io/sig-storage/csi-resizer:v1.8.0", + "registry.k8s.io/sig-storage/csi-snapshotter:v5.0.1", "registry.k8s.io/sig-storage/csi-snapshotter:v6.0.1", + "registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1", "registry.k8s.io/sig-storage/livenessprobe:v2.7.0", + "registry.k8s.io/sig-storage/livenessprobe:v2.10.0", ] diff --git a/magnum_cluster_api/image_utils.py b/magnum_cluster_api/image_utils.py index 059d74e7..ab695452 100644 --- a/magnum_cluster_api/image_utils.py +++ b/magnum_cluster_api/image_utils.py @@ -24,7 +24,7 @@ def update_manifest_images(cluster_uuid: str, file, repository=None, replacement docs = [] for doc in yaml.safe_load_all(data): # Fix container image paths - if doc["kind"] in ("DaemonSet", "Deployment"): + if doc["kind"] in ("DaemonSet", "Deployment", "StatefulSet"): for container in itertools.chain( doc["spec"]["template"]["spec"].get("initContainers", []), doc["spec"]["template"]["spec"]["containers"], From 59d35ba540d688aaca3f6874076b8fc62b2f36cd Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Sat, 27 May 2023 04:44:43 +1000 Subject: [PATCH 15/20] Use a stable version for nfs-csi --- magnum_cluster_api/cmd/image_loader.py | 7 +++-- .../manifests/nfs-csi/csi-nfs-controller.yaml | 25 +++-------------- .../manifests/nfs-csi/csi-nfs-node.yaml | 12 +++------ .../manifests/nfs-csi/rbac-csi-nfs.yaml | 27 ------------------- magnum_cluster_api/resources.py | 7 ----- tools/sync-nfs-csi-manifests | 3 ++- 6 files changed, 13 insertions(+), 68 deletions(-) diff --git a/magnum_cluster_api/cmd/image_loader.py b/magnum_cluster_api/cmd/image_loader.py index 6c206181..9ff9f182 100644 --- a/magnum_cluster_api/cmd/image_loader.py +++ b/magnum_cluster_api/cmd/image_loader.py @@ -29,7 +29,7 @@ "docker.io/k8scloudprovider/cinder-csi-plugin:v1.25.3", "docker.io/k8scloudprovider/manila-csi-plugin:v1.25.3", "docker.io/k8scloudprovider/openstack-cloud-controller-manager:v1.25.3", - "gcr.io/k8s-staging-sig-storage/nfsplugin:v4.2.0", + "registry.k8s.io/sig-storage/nfsplugin:v4.2.0", "registry.k8s.io/coredns/coredns:v1.8.6", "registry.k8s.io/coredns/coredns:v1.9.3", CONF.auto_scaling.v1_22_image, @@ -69,14 +69,13 @@ "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0", "registry.k8s.io/sig-storage/csi-provisioner:v3.0.0", "registry.k8s.io/sig-storage/csi-provisioner:v3.1.0", - "registry.k8s.io/sig-storage/csi-provisioner:v3.5.0", + "registry.k8s.io/sig-storage/csi-provisioner:v3.3.0", "registry.k8s.io/sig-storage/csi-resizer:v1.4.0", "registry.k8s.io/sig-storage/csi-resizer:v1.8.0", "registry.k8s.io/sig-storage/csi-snapshotter:v5.0.1", "registry.k8s.io/sig-storage/csi-snapshotter:v6.0.1", - "registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1", "registry.k8s.io/sig-storage/livenessprobe:v2.7.0", - "registry.k8s.io/sig-storage/livenessprobe:v2.10.0", + "registry.k8s.io/sig-storage/livenessprobe:v2.8.0", ] diff --git a/magnum_cluster_api/manifests/nfs-csi/csi-nfs-controller.yaml b/magnum_cluster_api/manifests/nfs-csi/csi-nfs-controller.yaml index 859768a3..1ca83423 100644 --- a/magnum_cluster_api/manifests/nfs-csi/csi-nfs-controller.yaml +++ b/magnum_cluster_api/manifests/nfs-csi/csi-nfs-controller.yaml @@ -23,7 +23,7 @@ spec: env: - name: ADDRESS value: /csi/csi.sock - image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0 name: csi-provisioner resources: limits: @@ -34,26 +34,12 @@ spec: volumeMounts: - mountPath: /csi name: socket-dir - - args: - - --v=5 - - --csi-address=$(ADDRESS) - - --leader-election-namespace=kube-system - - --leader-election - env: - - name: ADDRESS - value: /csi/csi.sock - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1 - imagePullPolicy: IfNotPresent - name: csi-snapshotter - volumeMounts: - - mountPath: /csi - name: socket-dir - args: - --csi-address=/csi/csi.sock - --probe-timeout=3s - --health-port=29652 - --v=2 - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.8.0 name: liveness-probe resources: limits: @@ -75,7 +61,7 @@ spec: fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix:///csi/csi.sock - image: gcr.io/k8s-staging-sig-storage/nfsplugin:canary + image: registry.k8s.io/sig-storage/nfsplugin:v4.2.0 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 @@ -108,14 +94,11 @@ spec: name: pods-mount-dir - mountPath: /csi name: socket-dir - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: Default hostNetwork: true nodeSelector: kubernetes.io/os: linux priorityClassName: system-cluster-critical - securityContext: - seccompProfile: - type: RuntimeDefault serviceAccountName: csi-nfs-controller-sa tolerations: - effect: NoSchedule diff --git a/magnum_cluster_api/manifests/nfs-csi/csi-nfs-node.yaml b/magnum_cluster_api/manifests/nfs-csi/csi-nfs-node.yaml index edaf92cc..9a543c94 100644 --- a/magnum_cluster_api/manifests/nfs-csi/csi-nfs-node.yaml +++ b/magnum_cluster_api/manifests/nfs-csi/csi-nfs-node.yaml @@ -18,7 +18,7 @@ spec: - --probe-timeout=3s - --health-port=29653 - --v=2 - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.8.0 name: liveness-probe resources: limits: @@ -40,7 +40,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.2 livenessProbe: exec: command: @@ -72,7 +72,7 @@ spec: fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix:///csi/csi.sock - image: gcr.io/k8s-staging-sig-storage/nfsplugin:canary + image: registry.k8s.io/sig-storage/nfsplugin:v4.2.0 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 @@ -105,14 +105,10 @@ spec: - mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional name: pods-mount-dir - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: Default hostNetwork: true nodeSelector: kubernetes.io/os: linux - priorityClassName: system-node-critical - securityContext: - seccompProfile: - type: RuntimeDefault serviceAccountName: csi-nfs-node-sa tolerations: - operator: Exists diff --git a/magnum_cluster_api/manifests/nfs-csi/rbac-csi-nfs.yaml b/magnum_cluster_api/manifests/nfs-csi/rbac-csi-nfs.yaml index 1ad3810e..2c304a17 100644 --- a/magnum_cluster_api/manifests/nfs-csi/rbac-csi-nfs.yaml +++ b/magnum_cluster_api/manifests/nfs-csi/rbac-csi-nfs.yaml @@ -42,33 +42,6 @@ rules: - get - list - watch -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshotclasses - - volumesnapshots - verbs: - - get - - list - - watch -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshotcontents - verbs: - - get - - list - - watch - - update - - patch -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshotcontents/status - verbs: - - get - - update - - patch - apiGroups: - '' resources: diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index 35265c73..0c677952 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -37,7 +37,6 @@ CALICO_TAG = "v3.24.2" CINDER_CSI_TAG = "v1.25.3" MANILA_CSI_TAG = "v1.25.3" -NFS_CSI_TAG = "v4.2.0" CLUSTER_CLASS_VERSION = pkg_resources.require("magnum_cluster_api")[0].version CLUSTER_CLASS_NAME = f"magnum-v{CLUSTER_CLASS_VERSION}" @@ -257,12 +256,6 @@ def get_object(self) -> pykube.ConfigMap: self.cluster.uuid, manifest, repository=repository, - replacements=[ - ( - "gcr.io/k8s-staging-sig-storage/nfsplugin:canary", - f"gcr.io/k8s-staging-sig-storage/nfsplugin:{NFS_CSI_TAG}", - ), - ], ) for manifest in glob.glob( os.path.join(manifests_path, "nfs-csi/*.yaml") diff --git a/tools/sync-nfs-csi-manifests b/tools/sync-nfs-csi-manifests index 2c152f6e..6eb97adc 100644 --- a/tools/sync-nfs-csi-manifests +++ b/tools/sync-nfs-csi-manifests @@ -28,10 +28,11 @@ MANIFESTS = set( "rbac-csi-nfs.yaml", ] ) +NFS_CSI_TAG = "v4.2.0" for manifest in MANIFESTS: - url = f"https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/deploy/{manifest}" # noqa E501 + url = f"https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/deploy/{NFS_CSI_TAG}/{manifest}" # noqa E501 docs = [] r = requests.get(url) From a30d9228b14e01337f13e3fb8b6cb20f5e1e9b87 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Mon, 29 May 2023 23:42:22 +1000 Subject: [PATCH 16/20] Set openstack auth credential in storageclass params --- magnum_cluster_api/resources.py | 24 ++++++++++++++++++++++++ magnum_cluster_api/utils.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index 0c677952..eb4764a1 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -251,6 +251,22 @@ def get_object(self) -> pykube.ConfigMap: ) data = { **data, + **{ + f"manila-csi-secret.yaml": yaml.dump( + { + "apiVersion": pykube.Secret.version, + "kind": pykube.Secret.kind, + "metadata": { + "name": "csi-manila-secrets", + "namespace": "kube-system", + }, + "stringData": utils.generate_manila_csi_cloud_config( + self.api, + self.cluster, + ), + }, + ) + }, **{ os.path.basename(manifest): image_utils.update_manifest_images( self.cluster.uuid, @@ -289,6 +305,14 @@ def get_object(self) -> pykube.ConfigMap: "provisioner": "manila.csi.openstack.org", "parameters": { "type": st.name, + "csi.storage.k8s.io/provisioner-secret-name": "csi-manila-secrets", + "csi.storage.k8s.io/provisioner-secret-namespace": "kube-system", + "csi.storage.k8s.io/controller-expand-secret-name": "csi-manila-secrets", + "csi.storage.k8s.io/controller-expand-secret-namespace": "kube-system", + "csi.storage.k8s.io/node-stage-secret-name": "csi-manila-secrets", + "csi.storage.k8s.io/node-stage-secret-namespace": "kube-system", + "csi.storage.k8s.io/node-publish-secret-name": "csi-manila-secrets", + "csi.storage.k8s.io/node-publish-secret-namespace": "kube-system", }, "reclaimPolicy": "Delete", "volumeBindingMode": "Immediate", diff --git a/magnum_cluster_api/utils.py b/magnum_cluster_api/utils.py index 2581bfcb..d2ef9d43 100644 --- a/magnum_cluster_api/utils.py +++ b/magnum_cluster_api/utils.py @@ -94,6 +94,34 @@ def generate_cloud_controller_manager_config( ) +def generate_manila_csi_cloud_config( + api: pykube.HTTPClient, + cluster: magnum_objects.Cluster, +) -> str: + """ + Generate coniguration of Openstack authentication for manila csi + """ + data = pykube.Secret.objects(api, namespace="magnum-system").get_by_name( + get_or_generate_cluster_api_cloud_config_secret_name(api, cluster) + ) + clouds_yaml = base64.decode_as_text(data.obj["data"]["clouds.yaml"]) + cloud_config = yaml.safe_load(clouds_yaml) + + return { + "os-authURL": cloud_config["clouds"]["default"]["auth"]["auth_url"], + "os-region": cloud_config["clouds"]["default"]["region_name"], + "os-applicationCredentialID": cloud_config["clouds"]["default"]["auth"][ + "application_credential_id" + ], + "os-applicationCredentialSecret": cloud_config["clouds"]["default"]["auth"][ + "application_credential_secret" + ], + "os-TLSInsecure": "false" + if cloud_config["clouds"]["default"]["verify"] + else "true", + } + + def get_kube_tag(cluster: magnum_objects.Cluster) -> str: return get_cluster_label(cluster, "kube_tag", "v1.25.3") From ddc07c5463f478ecbb5e6cb17ea62272e476212f Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Mon, 29 May 2023 23:56:32 +1000 Subject: [PATCH 17/20] Fix flake8 lint error --- magnum_cluster_api/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index eb4764a1..549a8466 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -252,7 +252,7 @@ def get_object(self) -> pykube.ConfigMap: data = { **data, **{ - f"manila-csi-secret.yaml": yaml.dump( + "manila-csi-secret.yaml": yaml.dump( { "apiVersion": pykube.Secret.version, "kind": pykube.Secret.kind, From 102200e3a323af36691fb4167c933eb9223030a6 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Tue, 30 May 2023 00:47:58 +1000 Subject: [PATCH 18/20] Create storageClasses if manila shared network id specified --- docs/labels.md | 15 ++++++++ magnum_cluster_api/resources.py | 63 +++++++++++++++++++-------------- 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/docs/labels.md b/docs/labels.md index 07b08db3..c7ee61f1 100644 --- a/docs/labels.md +++ b/docs/labels.md @@ -62,6 +62,21 @@ Default value: `v1.25.3` +### Manila + +* `manila_csi_plugin_tag` + + The version of the Manila CSI container image to use when bootstrapping the + cluster. + + Default value: `v1.25.3` + +* `manila_csi_share_network_id` + + Manila [share network](https://wiki.openstack.org/wiki/Manila/Concepts#share_network) ID. + + Default value: `None` + ## Kubernetes * `auto_healing_enabled` diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index 549a8466..b03a9051 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -249,6 +249,9 @@ def get_object(self) -> pykube.ConfigMap: csi_version = utils.get_cluster_label( self.cluster, "manila_csi_plugin_tag", MANILA_CSI_TAG ) + share_network_id = utils.get_cluster_label( + self.cluster, "manila_csi_share_network_id", None + ) data = { **data, **{ @@ -293,34 +296,40 @@ def get_object(self) -> pykube.ConfigMap: os.path.join(manifests_path, "manila-csi/*.yaml") ) }, - **{ - f"storageclass-manila-{st.name}.yaml": yaml.dump( - { - "apiVersion": objects.StorageClass.version, - "allowVolumeExpansion": True, - "kind": objects.StorageClass.kind, - "metadata": { - "name": "manila-%s" % st.name.lower(), - }, - "provisioner": "manila.csi.openstack.org", - "parameters": { - "type": st.name, - "csi.storage.k8s.io/provisioner-secret-name": "csi-manila-secrets", - "csi.storage.k8s.io/provisioner-secret-namespace": "kube-system", - "csi.storage.k8s.io/controller-expand-secret-name": "csi-manila-secrets", - "csi.storage.k8s.io/controller-expand-secret-namespace": "kube-system", - "csi.storage.k8s.io/node-stage-secret-name": "csi-manila-secrets", - "csi.storage.k8s.io/node-stage-secret-namespace": "kube-system", - "csi.storage.k8s.io/node-publish-secret-name": "csi-manila-secrets", - "csi.storage.k8s.io/node-publish-secret-namespace": "kube-system", - }, - "reclaimPolicy": "Delete", - "volumeBindingMode": "Immediate", - } - ) - for st in share_types - }, } + # NOTE: We only create StorageClasses if share_network_id specified. + if share_network_id: + data = { + **data, + **{ + f"storageclass-manila-{st.name}.yaml": yaml.dump( + { + "apiVersion": objects.StorageClass.version, + "allowVolumeExpansion": True, + "kind": objects.StorageClass.kind, + "metadata": { + "name": "manila-%s" % st.name.lower(), + }, + "provisioner": "manila.csi.openstack.org", + "parameters": { + "type": st.name, + "share_network_id": share_network_id, + "csi.storage.k8s.io/provisioner-secret-name": "csi-manila-secrets", + "csi.storage.k8s.io/provisioner-secret-namespace": "kube-system", + "csi.storage.k8s.io/controller-expand-secret-name": "csi-manila-secrets", + "csi.storage.k8s.io/controller-expand-secret-namespace": "kube-system", + "csi.storage.k8s.io/node-stage-secret-name": "csi-manila-secrets", + "csi.storage.k8s.io/node-stage-secret-namespace": "kube-system", + "csi.storage.k8s.io/node-publish-secret-name": "csi-manila-secrets", + "csi.storage.k8s.io/node-publish-secret-namespace": "kube-system", + }, + "reclaimPolicy": "Delete", + "volumeBindingMode": "Immediate", + } + ) + for st in share_types + }, + } return pykube.ConfigMap( self.api, From e47c9fa811ec800fde694c30d5d85f4bffa2f17b Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Tue, 30 May 2023 01:28:01 +1000 Subject: [PATCH 19/20] Fix image loader --- magnum_cluster_api/cmd/image_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magnum_cluster_api/cmd/image_loader.py b/magnum_cluster_api/cmd/image_loader.py index 9ff9f182..1ff9d038 100644 --- a/magnum_cluster_api/cmd/image_loader.py +++ b/magnum_cluster_api/cmd/image_loader.py @@ -66,7 +66,7 @@ "registry.k8s.io/sig-storage/csi-attacher:v3.4.0", "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.4.0", "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1", - "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0", + "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.2", "registry.k8s.io/sig-storage/csi-provisioner:v3.0.0", "registry.k8s.io/sig-storage/csi-provisioner:v3.1.0", "registry.k8s.io/sig-storage/csi-provisioner:v3.3.0", From 33786c4c5cc9af5c2be52f309a715ec97348e65e Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Wed, 31 May 2023 00:20:33 +1000 Subject: [PATCH 20/20] Fix share network id key name in sc --- magnum_cluster_api/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index b03a9051..844dbc2b 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -313,7 +313,7 @@ def get_object(self) -> pykube.ConfigMap: "provisioner": "manila.csi.openstack.org", "parameters": { "type": st.name, - "share_network_id": share_network_id, + "shareNetworkID": share_network_id, "csi.storage.k8s.io/provisioner-secret-name": "csi-manila-secrets", "csi.storage.k8s.io/provisioner-secret-namespace": "kube-system", "csi.storage.k8s.io/controller-expand-secret-name": "csi-manila-secrets",