Support multiple inventories and --limit

ansible-scylla-node/defaults/main.yml

@@ -89,7 +89,7 @@ install_only: false
 # Change here or override in vars
 scylla_nic: "{{ ansible_default_ipv4.interface }}"
 
-cluster_local_files_path: "{{ inventory_dir }}"
+manager_local_files_path: "{{ playbook_dir }}"
 
 # online|offline - offline will disable all the attepts to download packages
 # node_exporter will need to be installed manually
@@ -297,7 +297,7 @@ scylla_ssl:
 # Note that by default the {{ inventory_hostname }} is part of the localhost_system_key_directory. This allows you to define
 # different keys per host.
 # e.g.:
-# If your localhost_system_key_directory is set to "{{ inventory_dir }}/encryption_at_rest/{{ inventory_hostname }}/system_encryption_keys"
+# If your localhost_system_key_directory is set to "{{ playbook_dir }}/encryption_at_rest/{{ inventory_hostname }}/system_encryption_keys"
 # and you have the following file tree:
 #
 # ├── encryption_at_rest
@@ -317,7 +317,7 @@ scylla_ssl:
 # without the {{ inventory_hostname }} in it or simply copy the same key for all the node folders.
 
 handle_system_keys: false
-localhost_system_key_directory: "{{ inventory_dir }}/encryption_at_rest/{{ inventory_hostname }}/system_encryption_keys"
+localhost_system_key_directory: "{{ playbook_dir }}/encryption_at_rest/{{ inventory_hostname }}/system_encryption_keys"
 system_key_directory: "/etc/scylla/system_encryption_keys"
 
 # Table keys are used for encrypting SSTables. Depending on your key provider, this key is stored in different locations:
@@ -332,7 +332,7 @@ system_key_directory: "/etc/scylla/system_encryption_keys"
 # Note that by default the {{ inventory_hostname }} is part of the localhost_table_key_directory. This allows you to define
 # different keys per host.
 # e.g.:
-# If your localhost_table_key_directory is set to "{{ inventory_dir }}/encryption_at_rest/{{ inventory_hostname }}/table_encryption_keys"
+# If your localhost_table_key_directory is set to "{{ playbook_dir }}/encryption_at_rest/{{ inventory_hostname }}/table_encryption_keys"
 # and you have the following file tree:
 #
 # ├── encryption_at_rest
@@ -352,7 +352,7 @@ system_key_directory: "/etc/scylla/system_encryption_keys"
 # without the {{ inventory_hostname }} in it or simply copy the same key for all the node folders.
 
 handle_table_keys: false
-localhost_table_key_directory: "{{ inventory_dir }}/encryption_at_rest/{{ inventory_hostname }}/table_encryption_keys"
+localhost_table_key_directory: "{{ playbook_dir }}/encryption_at_rest/{{ inventory_hostname }}/table_encryption_keys"
 table_key_directory: "/etc/scylla/table_encryption_keys"
 
 # Encrypt system resources:

ansible-scylla-node/tasks/adjust_keyspace_replication.yml

@@ -1,14 +1,4 @@
 ---
-- name: Validate that all nodes are up before adjusting the replication
-  uri:
-    url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/failure_detector/endpoints/"
-    follow_redirects: none
-    method: GET
-  register: _result
-  until: _result.status == 200
-  retries: 10
-  delay: 1
-
 - name: Get datacenter name
   uri:
     url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/snitch/datacenter"
@@ -17,16 +7,27 @@
   until: _datacenter_out.status == 200
   retries: 5
   delay: 1
+  delegate_to: "{{ item }}"
+  loop: "{{ groups['scylla'] }}"
+  run_once: true
 
 - name: Prepare per DC replication_factor list
   set_fact:
-    dcs_to_rf: "{{ dcs_to_rf | default([]) + [\"'\" + hostvars[item]['_datacenter_out'].json + \"':\" + _keyspace_rf|string] }}"
-  loop: "{{ groups['scylla'] }}"
+    dcs_to_rf: "{{ dcs_to_rf | default([]) + [\"'\" + item.json + \"':\" + _keyspace_rf|string] }}"
+  loop: "{{ _datacenter_out.results }}"
   run_once: true
 
-- set_fact:
-    scylla_admin_username: "{{ scylla_admin_default_user }}"
-    scylla_admin_password: "{{ lookup('ini', scylla_admin_default_user, section='cql_credentials', allow_no_value=true, default=omit, file=inventory_file) }}"
+- name: Set scylla_admin_username and scylla_admin_password
+  block:
+    - name: Set scylla_admin_username
+      set_fact:
+        scylla_admin_username: "{{ scylla_admin_default_user }}"
+
+    - name: Look for cql_credentials in the inventories and read scylla_admin_password
+      include_tasks: read_admin_password.yml
+      vars:
+        iv_file: "{{ item }}"
+      loop: "{{ ansible_inventory_sources }}"
   when: scylla_admin_default_user is defined and 'cql_credentials' in groups
 
 - fail:

ansible-scylla-node/tasks/common.yml

@@ -79,7 +79,7 @@
 - name: Create a map from dc to its list of nodes
   set_fact:
     dc_to_node_list: "{{ dc_to_node_list | default({}) | combine( {hostvars[item]['dc']: (dc_to_node_list | default({}))[hostvars[item]['dc']] | default([]) + [item]} ) }}"
-  loop: "{{ groups['scylla'] }}"
+  loop: "{{ play_hosts }}"
   run_once: true
   when: scylla_io_probe_dc_aware|bool
 
@@ -328,13 +328,13 @@
     - name: Start seeders serially
       run_once: true
       include_tasks: start_one_node.yml
-      loop: "{{ groups['scylla'] }}"
+      loop: "{{ play_hosts }}"
       when: hostvars[item]['broadcast_address'] in scylla_seeds or item in scylla_seeds
 
     - name: Start scylla non-seeds nodes serially
       run_once: true
       include_tasks: start_one_node.yml
-      loop: "{{ groups['scylla'] }}"
+      loop: "{{ play_hosts }}"
       when:
         - item not in scylla_seeds
         - hostvars[item]['broadcast_address'] not in scylla_seeds
@@ -345,11 +345,17 @@
         host: "{{ scylla_api_address }}"
         timeout: 300
 
+    - name: Create a list with the broadcast addresses of the nodes in play_hosts
+      set_fact:
+        broadcast_address_list: "{{ broadcast_address_list | default([]) + [hostvars[item]['broadcast_address']] }}"
+      loop: "{{ play_hosts }}"
+      run_once: true
+
     - name: wait for the cluster to become healthy
       shell: |
-        nodetool status|grep -E '^UN|^UJ|^DN'|wc -l
+        nodetool status|grep -E '^UN|^UJ|^DN'| grep -Ew "{{ broadcast_address_list | join('|') }}" | wc -l
       register: node_count
-      until: node_count.stdout|int == ansible_play_batch|length
+      until: node_count.stdout|int == play_hosts|length
       retries: "{{ scylla_bootstrap_wait_time_sec|int }}"
       delay: 1
       when: full_inventory|bool
@@ -390,14 +396,26 @@
           {% if (_scylla_yaml_map.audit is defined and _scylla_yaml_map.audit == 'table') %}True{% else %}False{% endif %}
   run_once: true
 
+- name: Check if all nodes are up
+  block:
+    - name: Check if there are joining or down nodes
+      shell: |
+        nodetool status|grep -E '^UJ|^DN' | wc -l
+      register: node_count
+
+    - name: Set all_nodes_up as a fact
+      set_fact:
+        all_nodes_up: "{{ node_count.stdout|int == 0 }}"
+  run_once: true
+
 - name: Adjust replication for system_auth keyspace
   include_tasks: adjust_keyspace_replication.yml
   vars:
     _keyspace: "system_auth"
     _keyspace_replication_strategy: "{{ system_auth_replication_strategy }}"
     _keyspace_rf: "{{ system_auth_rf }}"
   when:
-    - start_scylla_service|bool
+    - all_nodes_up|bool
     - adjust_system_auth_replication is defined and adjust_system_auth_replication|bool
     - _authentication_enabled is defined and _authentication_enabled|bool
     - system_auth_rf is defined and system_auth_replication_strategy is defined
@@ -409,7 +427,7 @@
     _keyspace_replication_strategy: "{{ audit_replication_strategy }}"
     _keyspace_rf: "{{ audit_rf }}"
   when:
-    - start_scylla_service|bool
+    - all_nodes_up|bool
     - adjust_audit_replication is defined and adjust_audit_replication|bool
     - _audit_enabled is defined and _audit_enabled|bool
     - audit_rf is defined and audit_replication_strategy is defined
@@ -421,7 +439,7 @@
     _keyspace_replication_strategy: "{{ system_distributed_replication_strategy }}"
     _keyspace_rf: "{{ system_distributed_rf }}"
   when:
-    - start_scylla_service|bool
+    - all_nodes_up|bool
     - adjust_system_distributed_replication is defined and adjust_system_distributed_replication|bool
     - system_distributed_rf is defined and system_distributed_replication_strategy is defined
 
@@ -432,7 +450,7 @@
     _keyspace_replication_strategy: "{{ system_traces_replication_strategy }}"
     _keyspace_rf: "{{ system_traces_rf }}"
   when:
-    - start_scylla_service|bool
+    - all_nodes_up|bool
     - adjust_system_traces_replication is defined and adjust_system_traces_replication|bool
     - system_traces_rf is defined and system_traces_replication_strategy is defined
 

ansible-scylla-node/tasks/generate_tokens.yml

@@ -1,4 +1,48 @@
 ---
+- name: Get relevant addresses for nodes that are not in 'play_hosts'
+  block:
+    - name: check if scylla.yaml exists
+      stat:
+        path: /etc/scylla/scylla.yaml
+      register: _scylla_yaml_stat
+      delegate_to: "{{ item }}"
+      delegate_facts: true
+      when: item not in play_hosts
+      loop: "{{ groups['scylla'] }}"
+
+    - name: Set _scylla_yaml_exists as a fact
+      set_fact:
+        _scylla_yaml_exists: "{{ item.stat.exists }}"
+      when: item.item not in play_hosts
+      delegate_to: "{{ item.item }}"
+      delegate_facts: true
+      loop: "{{ _scylla_yaml_stat.results }}"
+
+    # Let's assume that a node can only be started if it has a scylla yaml
+    - name: Register scylla.yaml
+      command: cat /etc/scylla/scylla.yaml
+      register: _scylla_yaml_out
+      delegate_to: "{{ item }}"
+      when: "item not in play_hosts and hostvars[item]['_scylla_yaml_exists']|bool"
+      loop: "{{ groups['scylla'] }}"
+
+    - name: Set _scylla_yaml_map as a fact
+      set_fact:
+        _scylla_yaml_map: "{{ item.stdout | from_yaml }}"
+      delegate_to: "{{ item.item }}"
+      delegate_facts: true
+      when: item.stdout is defined
+      loop: "{{ _scylla_yaml_out.results }}"
+
+    - name: Set addresses as facts
+      set_fact:
+        rpc_address: "{{ hostvars[item]['_scylla_yaml_map'].rpc_address }}"
+        broadcast_address: "{{ hostvars[item]['_scylla_yaml_map'].broadcast_address }}"
+      delegate_to: "{{ item }}"
+      delegate_facts: true
+      when: item not in play_hosts and hostvars[item]['_scylla_yaml_exists']|bool
+      loop: "{{ groups['scylla'] }}"
+
 - name: Find the first node which is already bootstrapped, if any
   wait_for:
     port: 9042
@@ -7,7 +51,7 @@
   register: wait_for_cql_port_output
   ignore_errors: true
   delegate_to: "{{ item }}"
-  loop: "{{ groups['scylla'] }}"
+  loop: "{{ scylla_seeds }}"
   when: wait_for_cql_port_output is not defined or wait_for_cql_port_output.failed == True
 
 - name: Set the already bootstrapped node as a fact, if any
@@ -21,6 +65,7 @@
     node_list: "{{ node_list | default([]) + [hostvars[item]['broadcast_address']] }}"
     rack_list: "{{ rack_list | default([]) + [hostvars[item]['rack']] }}"
     dc_list: "{{ dc_list | default([]) + [hostvars[item]['dc'] + hostvars[item]['dc_suffix'] | default('')] }}"
+  when: hostvars[item]['broadcast_address'] is defined
   loop: "{{ groups['scylla'] }}"
 
 - name: Create a temporary tokens file
@@ -40,14 +85,15 @@
       retries: 5
       delay: 1
       delegate_to: "{{ bootstrapped_node }}"
+      when: hostvars[item]['broadcast_address'] is defined
       loop: "{{ groups['scylla'] }}"
 
     - name: Copy tokens to tmp file
       lineinfile:
         path: "{{ tokens_file.path }}"
         line: "{{ hostvars[item.item]['broadcast_address'] }}={{ item.json | map('int') | join(',') }}"
         create: yes
-      when: item.json|length > 0
+      when: hostvars[item.item]['broadcast_address'] is defined and item.json|length > 0
       delegate_to: localhost
       loop: "{{ _existing_tokens.results }}"
   when: bootstrapped_node is defined
@@ -77,4 +123,4 @@
   when: item[0].split('=')[0] == hostvars[item[1]]['broadcast_address']
   with_nested:
     - "{{ _new_tokens.stdout_lines }}"
-    - "{{ groups['scylla'] }}"
+    - "{{ play_hosts }}"

ansible-scylla-node/tasks/manager_agents.yml

@@ -2,7 +2,7 @@
 
 - name: check for pre-existing agent-token file
   stat:
-    path: "{{ cluster_local_files_path }}/scyllamgr_auth_token.txt"
+    path: "{{ manager_local_files_path }}/scyllamgr_auth_token.txt"
   register: token_file_stat
   run_once: true
   delegate_to: localhost
@@ -20,14 +20,14 @@
     copy:
       content: |
         {{ scyllamgr_auth_token.stdout }}
-      dest: "{{ cluster_local_files_path }}/scyllamgr_auth_token.txt"
+      dest: "{{ manager_local_files_path }}/scyllamgr_auth_token.txt"
     delegate_to: localhost
     run_once: true
   when: token_file_stat.stat.islnk is not defined
 
 - name: read the key from auth token file
   slurp:
-    src: "{{ cluster_local_files_path }}/scyllamgr_auth_token.txt"
+    src: "{{ manager_local_files_path }}/scyllamgr_auth_token.txt"
   register: b64auth_token
   delegate_to: localhost
   run_once: true

ansible-scylla-node/tasks/read_admin_password.yml

@@ -0,0 +1,6 @@
+---
+- name: Set password for scylla_admin
+  set_fact:
+    scylla_admin_password: "{{ lookup('ini', scylla_admin_username, section='cql_credentials', allow_no_value=true, default=omit, file=iv_file) }}"
+    inventory_with_credentials: "{{ iv_file }}"
+  ignore_errors: true

ansible-scylla-node/tasks/ssl.yml

@@ -1,6 +1,6 @@
 ---
 - set_fact:
-    _localhost_cert_path: "{{ scylla_ssl.localhost_cert_path | default(inventory_dir) }}"
+    _localhost_cert_path: "{{ scylla_ssl.localhost_cert_path | default(playbook_dir) }}"
 
 - name: For every node, check if crt file exists
   stat:

example-playbooks/manage_users/manage_users.yml

@@ -63,9 +63,13 @@
       when: _authentication_enabled|bool == false
       run_once: true
 
-    - name: Set password for scylla_admin
-      set_fact:
-        scylla_admin_password: "{{ lookup('ini', scylla_admin_username, section='cql_credentials', allow_no_value=true, default=omit, file=inventory_file) }}"
+    - name: Loop through inventories to find scylla_admin credentials
+      block:
+        - name: Try to read scylla_admin credentials in the current inventory
+          include_tasks: ../../ansible-scylla-node/tasks/read_admin_password.yml
+          vars:
+            iv_file: "{{ item }}"
+          loop: "{{ ansible_inventory_sources }}"
 
     - name: Create scylla_admin user
       import_tasks: create_user.yml
@@ -85,7 +89,7 @@
         admin_username: "{{ scylla_admin_username }}"
         admin_password: "{{ scylla_admin_password }}"
         username: "{{ item.key }}"
-        password: "{{ lookup('ini', item.key, section='cql_credentials', allow_no_value=true, default=omit, file=inventory_file) }}"
+        password: "{{ lookup('ini', item.key, section='cql_credentials', allow_no_value=true, default=omit, file=inventory_with_credentials) }}"
         permissions: "{{ item.value.permissions }}"
       loop: "{{ internal_users | dict2items }}"
       run_once: true