Fix make deploy on OpenShift

Author: empovit

.gitignore

@@ -27,8 +27,8 @@ testbin/*
 .vscode
 
 # Credentials and runtime environment
-hack/kubeconfig
-hack/.env
+kubeconfig
+.env
 
 # Bundle artifacts
 bundle.Dockerfile

Makefile

@@ -100,10 +100,13 @@ uninstall: manifests kustomize
 	$(KUSTOMIZE) build config/crd | kubectl delete -f -
 
 # Deploy gpu-operator in the configured Kubernetes cluster in ~/.kube/config
-deploy: manifests kustomize
+deploy: manifests generate-env kustomize
 	cd config/manager && $(KUSTOMIZE) edit set image gpu-operator=${IMAGE}
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
 
+generate-env:
+	./hack/prepare-env.sh
+
 # UnDeploy gpu-operator from the configured Kubernetes cluster in ~/.kube/config
 undeploy:
 	$(KUSTOMIZE) build config/default | kubectl delete -f -

config/manager/kustomization.yaml

@@ -8,6 +8,9 @@ configMapGenerator:
 - files:
   - controller_manager_config.yaml
   name: manager-config
+- envs:
+  - .env
+  name: gpu-operator-envs-cm
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:

config/manager/manager.yaml

@@ -3,13 +3,15 @@ kind: Namespace
 metadata:
   labels:
     control-plane: controller-manager
+    security.openshift.io/scc.podSecurityLabelSync: 'false'
     pod-security.kubernetes.io/enforce: privileged
   name: system
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: gpu-operator
+  namespace: system
   labels:
     app.kubernetes.io/component: gpu-operator
     app: gpu-operator
@@ -28,7 +30,7 @@ spec:
       serviceAccountName: gpu-operator
       containers:
       - name: gpu-operator
-        image: nvcr.io/nvidia/gpu-operator:1.7.0
+        image: gpu-operator
         args:
           - --leader-elect
         command:
@@ -63,6 +65,9 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: metadata.namespace
+        envFrom:
+          - configMapRef:
+              name: gpu-operator-envs-cm
         ports:
           - name: metrics
             containerPort: 8080

config/rbac/auth_proxy_role_binding.yaml

@@ -8,5 +8,5 @@ roleRef:
   name: proxy-role
 subjects:
 - kind: ServiceAccount
-  name: default
+  name: gpu-operator
   namespace: system

config/rbac/kustomization.yaml

@@ -1,4 +1,5 @@
 resources:
+- service_account.yaml
 - role.yaml
 - role_binding.yaml
 - leader_election_role.yaml

config/rbac/leader_election_role_binding.yaml

@@ -8,5 +8,5 @@ roleRef:
   name: leader-election-role
 subjects:
 - kind: ServiceAccount
-  name: default
+  name: gpu-operator
   namespace: system

config/rbac/role.yaml

@@ -55,6 +55,14 @@ rules:
   - pods
   verbs:
   - list
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - apps
   resources:
@@ -127,15 +135,27 @@ rules:
 - apiGroups:
   - monitoring.coreos.com
   resources:
-  - prometheusrule
+  - prometheusrules
   - servicemonitors
   verbs:
   - create
+  - delete
   - get
   - list
   - patch
   - update
   - watch
+- apiGroups:
+  - node.k8s.io
+  resources:
+  - runtimeclasses
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - update
+  - watch
 - apiGroups:
   - nvidia.com
   resources:
@@ -156,13 +176,7 @@ rules:
   - rolebindings
   - roles
   verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
+  - '*'
 - apiGroups:
   - route.openshift.io
   resources:
@@ -195,3 +209,11 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - security.openshift.io
+  resourceNames:
+  - privileged
+  resources:
+  - securitycontextconstraints
+  verbs:
+  - use

config/rbac/role_binding.yaml

@@ -8,5 +8,5 @@ roleRef:
   name: gpu-operator-role
 subjects:
 - kind: ServiceAccount
-  name: default
+  name: gpu-operator
   namespace: system

config/rbac/service_account.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gpu-operator
+  namespace: system

controllers/clusterpolicy_controller.go

@@ -61,16 +61,19 @@ type ClusterPolicyReconciler struct {
 // +kubebuilder:rbac:groups=nvidia.com,resources=*,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=config.openshift.io,resources=clusterversions;proxies,verbs=get;list;watch
 // +kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings;roles;rolebindings,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=use,resourceNames=privileged
+// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings;roles;rolebindings,verbs=*
 // +kubebuilder:rbac:groups="",resources=namespaces;serviceaccounts;pods;pods/eviction;services;services/finalizers;endpoints,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims;events;configmaps;secrets;nodes,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=apps,resources=deployments;daemonsets;replicasets;statefulsets,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=apps,resources=controllerrevisions,verbs=get;list;watch
-// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors;prometheusrule,verbs=get;list;watch;create;update;patch
+// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors;prometheusrules,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=get;list;watch;create
 // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch
 // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch
 // +kubebuilder:rbac:groups=image.openshift.io,resources=imagestreams,verbs=get;list;watch
+// +kubebuilder:rbac:groups=node.k8s.io,resources=runtimeclasses,verbs=get;list;create;update;watch;delete
+// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.

hack/README.md

@@ -1,18 +1,10 @@
 #! /bin/bash
 
+dest=config/manager/.env
+
 env=$(cat bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml \
-| yq \
-| jq '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | .name, "=", .value, ";"' -r )
+| yq '.spec.install.spec.deployments[].spec.template.spec.containers[].env[] | with_entries(select(.value != "OPERATOR_NAMESPACE"))' \
+| jq 'select( .name != null and .value != null) | .name, "=", .value, ";"' -r )
 env=${env//$'\n'/}
-echo $env > ./hack/.env
-sed -i 's/;/\n/g' ./hack/.env
-
-echo KUBECONFIG=${PWD}/hack/kubeconfig >> ./hack/.env
-echo OPERATOR_NAMESPACE=nvidia-gpu-operator >> ./hack/.env
-
-export KUBECONFIG=${PWD}/hack/kubeconfig
-export OPERATOR_NAMESPACE=nvidia-gpu-operator
-
-kubectl create namespace ${OPERATOR_NAMESPACE} 2>/dev/null || true
-kubectl apply -f ./config/crd/bases/nvidia.com_clusterpolicies.yaml
-kubectl apply -f ./config/samples/v1_clusterpolicy.yaml
+echo $env > ${dest}
+sed -i 's/;/\n/g' ${dest}