Repatch extended resource cmk.intel.com/exclusive-cores after kubelet restart.

Since 1.10 version of Kubernetes Kubelet sets extended resource
capacity to zero after it restarts. To repatch extended resource
cmk.intel.com/exclusive-cores "discovery" container added to
reconcile-report daemonset.

Signed-off-by: Liliia Butorina <[email protected]>

Author: LilyaBu

cmk.py

@@ -47,7 +47,7 @@
                                software.
   --cmk-cmd-list=<list>        Comma seperated list of CMK sub-commands to run
                                on each host
-                               [default: init,reconcile,install,discover,nodereport].
+                               [default: init,install,discover,rediscover,reconcile,nodereport].
   --cmk-img=<img>              CMK Docker image [default: cmk:v1.3.1].
   --cmk-img-pol=<pol>          Image pull policy for the CMK Docker image
                                [default: IfNotPresent].

docs/cli.md

@@ -1030,7 +1030,7 @@ $ docker run -it --volume=/etc/cmk:/etc/cmk:rw \
 ### `cmk uninstall`
 
 Removes `cmk` from a node. Uninstall process reverts `cmk cluster-init`:
- - deletes `cmk-reconcile-nodereport-pod-{node}` if present
+ - deletes `cmk-rediscover-reconcile-nodereport-pod-{node}` if present
  - removes `NodeReport` from Kubernetes ThirdPartyResources if present
  - removes `ReconcileReport` from Kubernetes ThirdPartyResources if present
  - removes cmk node label if present

docs/html/docs/cli.html

@@ -1027,7 +1027,7 @@ <h3>
 </h3>
 <p>Removes <code>cmk</code> from a node. Uninstall process reverts <code>cmk cluster-init</code>:</p>
 <ul>
-<li>deletes <code>cmk-reconcile-nodereport-pod-{node}</code> if present</li>
+<li>deletes <code>cmk-rediscover-reconcile-nodereport-pod-{node}</code> if present</li>
 <li>removes <code>NodeReport</code> from Kubernetes ThirdPartyResources if present</li>
 <li>removes <code>ReconcileReport</code> from Kubernetes ThirdPartyResources if present</li>
 <li>removes cmk node label if present</li>

docs/html/docs/operator.html

@@ -617,7 +617,7 @@ <h2>
 the recommended way to start troubleshooting is to look at the logs using <code>kubectl logs POD_NAME [CONTAINER_NAME] -f</code>.</p>
 <p>For example, assuming you ran the <a href="../resources/pods/cmk-cluster-init-pod.yaml">cmk-cluster-init-pod template</a> with default options, it
 should create two pods on each node named <code>cmk-init-install-discover-pod-&lt;node-name&gt;</code> and
-<code>cmk-reconcile-nodereport-&lt;node-name&gt;</code>, where <code>&lt;node-name&gt;</code> should be replaced with the name of the node.</p>
+<code>cmk-rediscover-reconcile-nodereport-&lt;node-name&gt;</code>, where <code>&lt;node-name&gt;</code> should be replaced with the name of the node.</p>
 <p>If you want to look at the logs from the container which ran the <code>discover</code> subcommand in the pod, you can use
 <code>kubectl logs -f cmk-init-install-discover-pod-&lt;node-name&gt; discover</code></p>
 <p>If you want to look at the logs from the container which ran the <code>reconcile</code> subcommand in the pod, you can use

docs/operator.md

@@ -567,13 +567,13 @@ the recommended way to start troubleshooting is to look at the logs using `kubec
 
 For example, assuming you ran the [cmk-cluster-init-pod template][cluster-init-template] with default options, it
 should create two pods on each node named `cmk-init-install-discover-pod-<node-name>` and
-`cmk-reconcile-nodereport-<node-name>`, where `<node-name>` should be replaced with the name of the node.
+`cmk-rediscover-reconcile-nodereport-<node-name>`, where `<node-name>` should be replaced with the name of the node.
 
 If you want to look at the logs from the container which ran the `discover` subcommand in the pod, you can use
 `kubectl logs -f cmk-init-install-discover-pod-<node-name> discover`
 
 If you want to look at the logs from the container which ran the `reconcile` subcommand in the pod, you can use
-`kubectl logs -f cmk-reconcile-nodereport-pod-<node-name> reconcile`
+`kubectl logs -f cmk-rediscover-reconcile-nodereport-pod-<node-name> reconcile`
 
 If you want to remove `cmk` use `cmk-uninstall-pod.yaml`. [nodeSelector](https://kubernetes.io/docs/user-guide/node-selection)
 can help to fine-grain the deletion for specific node.

intel/clusterinit.py

@@ -36,7 +36,7 @@ def cluster_init(host_list, all_hosts, cmd_list, cmk_img, cmk_img_pol,
 
     # Check if all the flag values passed are valid.
     # Check if cmk_cmd_list is valid.
-    valid_cmd_list = ["init", "discover", "install", "reconcile", "nodereport"]
+    valid_cmd_list = ["init", "discover", "install", "rediscover", "reconcile", "nodereport"]
     for cmk_cmd in cmk_cmd_list:
         if cmk_cmd not in valid_cmd_list:
             raise RuntimeError("CMK command should be one of {}"
@@ -147,6 +147,8 @@ def run_cmd_pods(cmd_list, cmd_init_list, cmk_img, cmk_img_pol, conf_dir,
                 args = "/cmk/cmk.py isolate --pool=infra /cmk/cmk.py -- reconcile --interval=5 --publish"  # noqa: E501
             elif cmd == "nodereport":
                 args = "/cmk/cmk.py isolate --pool=infra /cmk/cmk.py -- node-report --interval=5 --publish"  # noqa: E501
+            elif cmd == "rediscover":
+                args = "/cmk/cmk.py isolate --pool=infra /cmk/cmk.py -- discover; sleep infinity" # noqa: E501
 
             update_pod_with_container(pod, cmd, cmk_img, cmk_img_pol, args)
     elif cmd_init_list:
@@ -178,10 +180,10 @@ def run_cmd_pods(cmd_list, cmd_init_list, cmk_img, cmk_img_pol, conf_dir,
 
     for node_name in cmk_node_list:
         if cmd_list:
-            update_pod_with_node_details(pod, node_name, cmd_list)
+            update_pod_with_node_details(pod, node_name, cmd_list, "ds")
             daemon_set = k8s.ds_from(pod=pod)
         elif cmd_init_list:
-            update_pod_with_node_details(pod, node_name, cmd_init_list)
+            update_pod_with_node_details(pod, node_name, cmd_init_list, "pod")
 
         try:
             if cmd_list:
@@ -315,8 +317,9 @@ def wait_for_pod_phase(pod_name, phase_name):
             sys.exit(1)
 
         for pod in pod_list_resp["items"]:
-            if ("metadata" in pod) and ("name" in pod["metadata"]) \
-                    and pod_name in pod["metadata"]["name"]:
+            if ("metadata" in pod) and ("labels" in pod["metadata"]) \
+                    and ("podname" in pod["metadata"]["labels"]) \
+                    and (pod_name == pod["metadata"]["labels"]["podname"]):
                 if pod["status"]["phase"] == phase_name:
                     wait = False
                     break
@@ -333,10 +336,13 @@ def update_pod(pod, restart_pol, conf_dir, install_dir, serviceaccount):
     pod["spec"]["volumes"][2]["hostPath"]["path"] = install_dir
 
 
-def update_pod_with_node_details(pod, node_name, cmd_list):
+def update_pod_with_node_details(pod, node_name, cmd_list, res_type):
     pod["spec"]["nodeName"] = node_name
-    pod_name = "cmk-{}-pod-{}".format("-".join(cmd_list), node_name)
+    pod_name = "cmk-{}-{}-{}".format("-".join(cmd_list), res_type, node_name)
     pod["metadata"]["name"] = pod_name
+    # name max length is 63, so move to labels key-value
+    pod["metadata"]["labels"] = {"podname": pod_name}
+    logging.info("Created pod name: {}".format(pod_name))
 
 
 def update_pod_with_pull_secret(pod, pull_secret):

intel/k8s.py

@@ -69,7 +69,9 @@ def ds_from(pod):
                 "metadata": {
                     "labels": {
                         "app":
-                            pod["metadata"]["name"].replace("pod", "ds")
+                            pod["metadata"]["name"].replace("pod", "ds"),
+                        "podname":
+                            pod["metadata"]["labels"]["podname"]
                     }
                 },
                 "spec": pod["spec"]

intel/uninstall.py

@@ -32,7 +32,7 @@
 def uninstall(install_dir, conf_dir, namespace):
     delete_cmk_pod("cmk-init-install-discover-pod", namespace,
                    postfix=os.getenv("NODE_NAME"))
-    delete_cmk_pod("cmk-reconcile-nodereport-ds", namespace,
+    delete_cmk_pod("cmk-rediscover-reconcile-nodereport-ds", namespace,
                    postfix=os.getenv("NODE_NAME"))
 
     delete_cmk_pod("cmk-node-report-ds-all", namespace)