Support CDP Base 7.1.8 for ECS autodeployment on EC2

galaxy.yml

@@ -23,7 +23,7 @@ namespace: cloudera
 name: cluster
 
 # The version of the collection. Must be compatible with semantic versioning
-version: 3.4.0
+version: 4.0.0-alpha1
 
 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection
 readme: README.md

roles/cloudera_manager/config/filter_plugins/filters.py → roles/cloudera_manager/api_client/handlers/main.yml

@@ -1,4 +1,3 @@
-#!/usr/bin/python
 # Copyright 2021 Cloudera, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,17 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+---
 
-class FilterModule(object):
-
-  def filters(self):
-    return {
-      'filter_null_configs': self.filter_null_configs
-    }
-
-  def filter_null_configs(self, configs, existing_configs):
-    filtered_configs = dict(configs)
-    for item, value in configs.items():
-      if item not in existing_configs and not value:
-        del filtered_configs[item]
-    return filtered_configs
+- name: restart cloudera management service
+  cm_api:
+    endpoint: /cm/service/commands/restart
+    method: POST
+    timeout: "{{ cluster_restart_timeout | default(3000) }}"

roles/cloudera_manager/autotls/files/cert.py_patch

@@ -0,0 +1,11 @@
+--- cert.py	2020-12-02 00:54:05.000000000 +0100
++++ cert.py_2	2021-02-18 09:09:38.095192730 +0100
+@@ -1949,7 +1949,7 @@
+       LOG.info("Could not find JKS truststore at location: %s. Converting "
+                "PEM truststore to JKS." % cluster_ca_jks)
+       generate_truststore(self.cfg.keytool, cluster_ca_jks, truststore_password,
+-                          cluster_ca_pem)
++                          cluster_ca_pem, self.cfg.keystore_type)
+
+     global_ca_pem = self.trust_files[GLOBAL_TLS_SET][PEM_TLS_TYPE]
+     copied_cluster_to_global = False

roles/cloudera_manager/autotls/tasks/main.yml

@@ -23,23 +23,48 @@
     msg: This playbook requires Cloudera Manager 7.1+
   when: response.json.version is version('7.1', '<')
 
+- name: Patch Cloudera Manager older than 7.3
+  include_tasks:
+    file: patch_old_cm
+  when: response.json.version is version('7.3.0', '<')
+
+- name: Check if password or key is used to connect to machines
+  set_fact:
+    use_password: "{{ true if node_password is defined and node_password|length > 0 else false }}"
+
+- name: DEBUG Auto-TLS using password
+  debug:
+    msg: "{{ lookup('template', 'auto-tls.json') }}"
+  when: use_password and debug | default(false)
+
 - name: Enable Auto-TLS
-  cloudera.cluster.cm_api:
-    endpoint: /cm/commands/generateCmca
+  cm_api:
+    endpoint: "/cm/commands/generateCmca"
     method: POST
-    body: "{{ lookup('template', 'request.j2', convert_data=False) }}"
+    body: "{{ lookup('template', 'auto-tls.json') }}"
+    timeout: 360
+  ignore_errors: true
+  when: use_password
 
-- name: Restart Cloudera Manager Server
-  service:
-    name: cloudera-scm-server
-    state: restarted
-  notify:
-    - wait cloudera-scm-server
+- name: Set node_key on one line
+  set_fact:
+    node_key_one_line: "{{ lookup('file', '~/node_key' ) | replace('\n', '\\n') | replace('\"', '\\\"' ) }}"
+  when: not use_password
 
-- name: Wait for Cloudera Manager Server to come back up
-  meta: flush_handlers
+- name: DEBUG Auto-TLS using key
+  debug:
+    msg: "{{ lookup('template', 'auto-tls-key.json') }}"
+  when: not use_password
 
-- name: Restart Cloudera Management Service
-  cloudera.cluster.cm_api:
-    endpoint: /cm/service/commands/restart
+- name: Enable Auto-TLS
+  cm_api:
+    endpoint: "/cm/commands/generateCmca"
     method: POST
+    body: "{{ lookup('template', 'auto-tls-key.json') }}"
+  ignore_errors: true
+  when: not use_password
+  notify:
+    - restart cloudera-scm-server
+    - restart cloudera management service
+    - restart cloudera-scm-agent
+

roles/cloudera_manager/autotls/tasks/patch_old_cm.yml

@@ -0,0 +1,16 @@
+---
+- name: Copy patch to machines
+  copy:
+    src: "{{ role_path}}/files/cert.py_patch"
+    dest: /opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/tools/cert.py_patch
+    owner: cloudera-scm
+    group: cloudera-scm
+    mode: '0644'
+
+- name: Backup cert.py
+  shell: cp /opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/tools/cert.py /opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/tools/cert.py.backup
+
+- name: Fix cert.py
+  ansible.posix.patch:
+    src: "{{ role_path}}/patch/cert.py_patch"
+    dest: /opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/tools/cert.py

roles/cloudera_manager/autotls/templates/auto-tls-key.json

@@ -0,0 +1,9 @@
+{
+    "customCA" : false,
+    "configureAllServices" : "true",
+    "sshPort" : 22,
+    {% if freeipa_activated %}"trustedCaCerts" : "/etc/ipa/ca.crt",{% endif %}
+    "userName" : "root",
+    "privateKey": "{{ node_key_one_line }}"
+} 
+

roles/cloudera_manager/autotls/templates/auto-tls.json

@@ -0,0 +1,9 @@
+{
+    "customCA" : false,
+    "configureAllServices" : "true",
+    "sshPort" : 22,
+    {% if freeipa_activated %}"trustedCaCerts" : "/etc/ipa/ca.crt",{% endif %}
+    "userName" : "root",
+    "password": "{{ node_password }}"
+} 
+

roles/cloudera_manager/cms_tls/files/cms_keystore_tls.json

@@ -0,0 +1,16 @@
+{
+    "items": [
+      {
+        "name": "ssl_server_keystore_location",
+        "value": "{{CM_AUTO_TLS}}"
+      },
+      {
+        "name": "ssl_server_keystore_password",
+        "value": "{{CM_AUTO_TLS}}"
+      },
+      {
+        "name": "ssl_enabled",
+        "value": "true"
+      }
+    ]
+  }

roles/cloudera_manager/cms_tls/files/cms_navigator_keystore_tls.json

@@ -0,0 +1,28 @@
+{
+    "items": [
+      {
+        "name": "navigator_truststore_file",
+        "value": "{{CM_AUTO_TLS}}"
+      },
+      {
+        "name": "navigator_truststore_password",
+        "value": "{{CM_AUTO_TLS}}"
+      },
+      {
+        "name": "ssl_server_keystore_location",
+        "value": "{{CM_AUTO_TLS}}"
+      },
+      {
+        "name": "ssl_server_keystore_password",
+        "value": "{{CM_AUTO_TLS}}"
+      },
+      {
+        "name": "ssl_server_keystore_keypassword",
+        "value": "{{CM_AUTO_TLS}}"
+      },
+      {
+        "name": "ssl_enabled",
+        "value": "true"
+      }
+    ]
+  }

roles/cloudera_manager/cms_tls/files/cms_navigator_metaserver_keystore_tls.json

@@ -0,0 +1,20 @@
+{
+    "items": [
+      {
+        "name": "ssl_server_keystore_location",
+        "value": "{{CM_AUTO_TLS}}"
+      },
+      {
+        "name": "ssl_server_keystore_password",
+        "value": "{{CM_AUTO_TLS}}"
+      },
+      {
+        "name": "ssl_server_keystore_keypassword",
+        "value": "{{CM_AUTO_TLS}}"
+      },
+      {
+        "name": "ssl_enabled",
+        "value": "true"
+      }
+    ]
+  }

roles/cloudera_manager/cms_tls/files/cms_truststore_tls.json

@@ -0,0 +1,12 @@
+{
+    "items": [
+      {
+        "name": "ssl_client_truststore_location",
+        "value": "{{CM_AUTO_TLS}}"
+      },
+      {
+        "name": "ssl_client_truststore_password",
+        "value": "{{CM_AUTO_TLS}}"
+      }
+    ]
+  }

roles/cloudera_manager/cms_tls/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: cloudera.cluster.cloudera_manager.api_client

roles/cloudera_manager/cms_tls/tasks/main.yml

@@ -0,0 +1,40 @@
+---
+- name: Setup TLS for Activity Monitor
+  cm_api:
+    method: PUT 
+    endpoint: /cm/service/roleConfigGroups/mgmt-ACTIVITYMONITOR-BASE/config
+    body: "{{ lookup('file', 'cms_keystore_tls.json', convert_data=False) }}"
+
+- name: Setup TLS for Host Monitor
+  cm_api:
+    method: PUT 
+    endpoint: /cm/service/roleConfigGroups/mgmt-HOSTMONITOR-BASE/config
+    body: "{{ lookup('file', 'cms_keystore_tls.json', convert_data=False) }}"
+
+- name: Setup TLS for Service Monitor
+  cm_api:
+    method: PUT 
+    endpoint: /cm/service/roleConfigGroups/mgmt-SERVICEMONITOR-BASE/config
+    body: "{{ lookup('file', 'cms_keystore_tls.json', convert_data=False) }}"
+
+- name: Setup TLS for Navigator
+  cm_api:
+    method: PUT 
+    endpoint: /cm/service/roleConfigGroups/mgmt-NAVIGATOR-BASE/config
+    body: "{{ lookup('file', 'cms_navigator_keystore_tls.json', convert_data=False) }}"
+  when: cloudera_manager_version is version('7.0.0','<')
+
+- name: Setup TLS for Navigator Meta Server 
+  cm_api:
+    method: PUT 
+    endpoint: /cm/service/roleConfigGroups/mgmt-NAVIGATORMETASERVER-BASE/config
+    body: "{{ lookup('file', 'cms_navigator_metaserver_keystore_tls.json', convert_data=False) }}"
+  when: cloudera_manager_version is version('7.0.0','<')
+
+- name: Setup TLS for CMS
+  cm_api:
+    method: PUT 
+    endpoint: /cm/service/config
+    body: "{{ lookup('file', 'cms_truststore_tls.json', convert_data=False) }}"
+  notify:
+    - restart cloudera management service

roles/cloudera_manager/config/meta/main.yml

@@ -14,6 +14,6 @@
 
 ---
 dependencies:
-  - role: cloudera_manager/api_client
+  - role: cloudera.cluster.cloudera_manager.api_client
 
 

roles/cloudera_manager/external_auth/defaults/main.yml

@@ -13,7 +13,31 @@
 # limitations under the License.
 
 ---
-
 cloudera_manager_external_auth:
+  provider: "{{ 'FreeIPA' if freeipa_activated == true else omit }}"
   external_first: no
   external_only: no
+  external_set: "{{ 'yes' if freeipa_activated == true else 'no' }}"
+  role_mappings: "{{ default_free_ipa_role_mappings if freeipa_activated == true else omit }}"
+
+default_free_ipa_role_mappings:
+  - group: admins
+    roles: [ ROLE_ADMIN ]
+  - group: auditors
+    roles: [ ROLE_AUDITOR ]
+  - group: users
+    roles: [ ROLE_USER ]
+
+auth_providers:
+  FreeIPA:
+    type: LDAP
+    ldap_url: "{{ ipa_ldap_url }}"
+    ldap_base_dn:
+    ldap_bind_user_dn: "{{ ipa_ldap_user_bind_dn }}"
+    ldap_bind_password: "{{ ipa_ldap_user_bind_password }}"
+    ldap_search_base:
+      user: "{{ ipa_ldap_user_search_base }}"
+      group: "{{ ipa_ldap_group_search_base }}"
+    ldap_search_filter:
+      user: "{{ ipa_ldap_user_search_filter }}"
+      group: "{{ ipa_ldap_user_group_filter }}"

roles/cloudera_manager/external_auth/meta/main.yml

@@ -16,3 +16,5 @@
 
 dependencies:
   - role: cloudera.cluster.cloudera_manager.api_client
+  - role: cloudera.cluster.infrastructure.krb5_common
+  - role: cloudera.cluster.cloudera_manager.repo

roles/cloudera_manager/kerberos/tasks/main.yml

@@ -26,6 +26,9 @@
     endpoint: /cm/commands/importAdminCredentials?username={{ krb5_kdc_admin_user | urlencode }}&password={{ krb5_kdc_admin_password | urlencode }}
     method: POST
   register: result
+  failed_when:
+    - result is failed
+    - "'already exists' not in result.content"
   until: result is not failed
   retries: 3
   delay: 10

roles/cloudera_manager/repo/defaults/main.yml

@@ -15,5 +15,9 @@
 ---
 cloudera_archive_base_url: https://archive.cloudera.com
 cloudera_manager_version: 7.4.4
+cloudera_manager_distro_name: "{{ ansible_os_family | lower }}"
+cloudera_manager_distro_version: "{{ ansible_distribution_major_version }}"
 
-install_repo_on_host: yes
+install_repo_on_host: yes
+
+set_custom_repo_as_archive_base_url: "{{ use_custom_repo_as_archive_base_url | default(True) }}"

roles/cloudera_manager/repo/tasks/main-RedHat.yml

@@ -28,4 +28,4 @@
 - name: yum-clean-metadata
   command: yum clean metadata
   args:
-    warn: no
+    warn: no