YDBOPS-12647: New inventory and some fixes

README.md

@@ -57,12 +57,12 @@ The meaning and format of the variables used are specified in the table below.
 | `ydb_disks` | Disk layout of storage nodes, defined as `ydbd_static` in the hosts file. Defined as list of structures having the following fields:<br/> `name` - physical device name (like `/dev/sdb` or `/dev/vdb`);<br/> `label` - the desired YDB data partition label, as used in the cluster configuration file (like `ydb_disk_1`) |
 | `ydb_dynnodes` | Set of dynamic nodes to be ran on each host listed as `ydbd_dynamic` in the hosts file. Defined as list of structures having the following fields:<br/> `dbname` - name of the YDB database handled by the corresponding dynamic node;<br/> `instance` - dynamic node service instance name, allowing to distinguish between multiple dynamic nodes for the same database running in the same host;<br/> `offset` - integer number `0-N`, used as the offset for the standard network port numbers (`0` means using the standard ports). |
 | `ydb_brokers` | List of host names running the YDB static nodes, exactly 3 (three) host names must be specified |
-| `ydb_storage_node_cores` | Number of cores to be used by thread pools of the storage nodes (alias: `ydb_cores_static`) |
-| `ydb_database_node_cores` | Number of cores to be used by thread pools of the database nodes (alias: `ydb_cores_dynamic`) |
+| `ydb_cores_static` | Number of cores to be used by thread pools of the storage nodes |
+| `ydb_cores_dynamic` | Number of cores to be used by thread pools of the database nodes |
 | `ydb_config_v2` | Controls initialization method and configuration management: if `true` uses YDB configuration V2 with `node init` + `bootstrap` and `--config-dir` arg, if `false` uses configuration V1 with `actor_system_config` injection and `--yaml-config` arg. If not set, automatically determined based on YDB version (`true` for version 25.1 and above) |
-| `ydb_database_name` | Database name, for database creation, dynamic nodes deployment and dynamic nodes rolling restart (alias: `ydb_dbname`) |
+| `ydb_dbname` | Database name, for database creation, dynamic nodes deployment and dynamic nodes rolling restart |
 | `ydb_pool_kind` | YDB default storage pool kind, as specified in the static nodes configuration file in the `storage_pool_types.kind` field |
-| `ydb_database_storage_groups` | Initial number of storage groups in the newly created database (alias: `ydb_database_groups`) |
+| `ydb_database_groups` | Initial number of storage groups in the newly created database |
 | `ydb_dynnode_restart_sleep_seconds` | Number of seconds to sleep after startup of each dynamic node during the rolling restart. |
 
 ### Installing the YDB cluster using Ansible

filter_plugins/yaml_filters.py

@@ -5,7 +5,13 @@
 
 from ansible_collections.ydb_platform.ydb.plugins.filter.yaml_filters import ydb_config_to_yaml, FilterModule as OrigFilterModule
 
-
+DOCUMENTATION = r'''
+    name: yaml_filters
+    plugin_type: filter
+    short_description: short desc
+    description: |
+        long description
+'''
 
 class FilterModule(object):
     """

galaxy.yml

@@ -8,7 +8,7 @@ namespace: ydb_platform
 name: ydb
 
 # The version of the collection. Must be compatible with semantic versioning
-version: 1.2.0
+version: 1.2.5
 
 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection
 readme: README.md
@@ -20,7 +20,6 @@ authors:
 - Eugene Arbatsky <[email protected]> elabpro
 - Maksim Zinal <[email protected]> zinal
 
-
 ### OPTIONAL but strongly recommended
 # A short summary description of the collection
 description: Collection for YDB cluster management.

meta/runtime.yaml

@@ -50,3 +50,9 @@
 #   group_name:
 #     - module1
 #     - module2
+plugin_routing:
+  inventory:
+    ydb_inventory:
+      name: ydb_platform.ydb.ydb_inventory
+      import_redirection:
+        redirect: plugins.inventory.ydb_inventory

playbooks/create_database.yaml

@@ -16,7 +16,7 @@
 # ansible-playbook ydb_platform.ydb.create_database
 #
 # Create DB with new name
-# ansible-playbook ydb_platform.ydb.create_database --extra-vars "ydb_database_name=newdbname"
+# ansible-playbook ydb_platform.ydb.create_database --extra-vars "ydb_dbname=newdbname"
 #
 # WARNING: There will be no dynamic nodes for the new database. You have to create them by using
 # playbook `install_dynamic`

playbooks/healthcheck.yaml

@@ -2,6 +2,8 @@
 - name: Healthcheck cluster
   hosts: "{{ ansible_play_hosts | default(groups['ydbd_static']) }}"
   run_once: true
+  vars:
+    ydb_host_user: ydb
   tasks:
 
     - name: prepare
@@ -23,7 +25,7 @@
       retries: 25
       delay: 20
       become: true
-      become_user: ydb
+      become_user: "{{ ydb_host_user }}"
       tags:
         - token
 
@@ -36,7 +38,7 @@
         token: "{{ ydb_credentials.token }}"
       any_errors_fatal: true
       become: true
-      become_user: ydb
+      become_user: "{{ ydb_host_user }}"
       tags:
         - discovery
 
@@ -54,6 +56,7 @@
       retries: 30
       delay: 10
       become: true
-      become_user: ydb
+      become_user: "{{ ydb_host_user }}"
       tags:
         - healthcheck
+      when: "(ydb_version is regex(\"\\d+\\..*\") or ydb_version is regex(\"\\d+-.*\")) and ydb_version is version('25.1', '<')" # Hack until YDB HC will handle 3-nodes-mirror-3-dc in 25.*

playbooks/install_dynamic.yaml

@@ -2,7 +2,7 @@
 
 
 # PLAYBOOK: ydb_platform.ydb.install_dynamic
-# PROPOSE : This playbook is designed to create YDB database in YDB cluster.
+# PROPOSE : This playbook is designed to install YDB dynamic(database) nodes in YDB cluster.
 #           
 # HOW TO USE:
 #     Use https://github.com/ydb-platform/ydb-ansible-examples as an example to prepare Ansible

playbooks/update_config.yaml

@@ -20,11 +20,11 @@
 
   - ansible.builtin.assert:
       that:
-        - "{{ item.var1 is defined or item.var2 is defined }}"
-      fail_msg: "Either {{ item.var1 }} or {{ item.var2 }} variable is required"
+        - "{{ item.var1 is defined}}"
+      fail_msg: "Either {{ item.var1 }} variable is required"
     loop:
-      - { var1: "ydb_storage_node_cores", var2: "ydb_cores_static" }
-      - { var1: "ydb_database_node_cores", var2: "ydb_cores_dynamic" }
+      - { var1: "ydb_cores_static" }
+      - { var1: "ydb_cores_dynamic" }
   tags:
     - storage
     - static
@@ -144,37 +144,9 @@
         retries: 30
         delay: 15
 
-      - name: check if ydb_config is a map
-        set_fact:
-          ydb_config_is_map: "{{ ydb_config is mapping }}"
-        run_once: true
-
-      - name: handle ydb_config as map
-        block:
-          - name: parse ansible configuration from map
-            set_fact:
-              parsed_ansible_config: "{{ ydb_config }}"
-            run_once: true
-        when: ydb_config_is_map
-
-      - name: handle ydb_config as file path
-        block:
-          - name: read ansible configuration file content
-            slurp:
-              src: "{{ ydb_config }}"
-            run_once: true
-            register: ansible_config_content
-            delegate_to: localhost
-
-          - name: parse ansible configuration content
-            set_fact:
-              parsed_ansible_config: "{{ ansible_config_content.content | b64decode | from_yaml }}"
-            run_once: true
-        when: not ydb_config_is_map
-
       - name: apply patch_config_v2 to configuration
         ydb_platform.ydb.patch_config:
-          config: "{{ parsed_ansible_config }}"
+          config: "{{ ydb_config_dict }}"
           output_file: "{{ ydb_temp_config_path }}"
           hostvars: "{{ hostvars }}"
           ydb_disks: "{{ ydb_disks }}"
@@ -199,12 +171,12 @@
         ydb_platform.ydb.apply_config_selectors:
           config: "{{ ydb_temp_config_path }}"
           output_file: "{{ ydb_temp_config_path }}"
-          database_cores: "{{ ydb_database_node_cores | default(ydb_cores_dynamic) | default(omit) }}"
-          storage_cores: "{{ ydb_storage_node_cores | default(ydb_cores_static) | default(omit) }}"
+          database_cores: "{{ ydb_cores_dynamic | default(omit) }}"
+          storage_cores: "{{ydb_cores_static | default(omit) }}"
         become: true
         become_user: ydb
         run_once: true
-        when: ydb_database_node_cores is defined or ydb_storage_node_cores is defined or ydb_cores_dynamic is defined or ydb_cores_static is defined
+        when: ydb_cores_dynamic is defined or ydb_cores_static is defined
         register: selectors_result
         until: selectors_result is succeeded
         retries: 5
@@ -293,7 +265,7 @@
       - no_restart
 
   - name: update static node configuration file (V1 configuration only)
-    command: "{{ ydb_dir }}/home/update_config_file.sh ydbd-config.yaml ydbd-config-static.yaml STORAGE {{ ydb_storage_node_cores | default(ydb_cores_static, true) }}"
+    command: "{{ ydb_dir }}/home/update_config_file.sh ydbd-config.yaml ydbd-config-static.yaml STORAGE {{ ydb_cores_static | default(ydb_cores_static, true) }}"
     changed_when: false
     when: not ydb_config_v2 | default(false)
     tags:
@@ -302,7 +274,7 @@
       - no_restart
 
   - name: create dynamic node configuration file (V1 configuration only)
-    command: "{{ ydb_dir }}/home/update_config_file.sh ydbd-config.yaml ydbd-config-dynamic.yaml COMPUTE {{ ydb_database_node_cores | default(ydb_cores_dynamic, true) }}"
+    command: "{{ ydb_dir }}/home/update_config_file.sh ydbd-config.yaml ydbd-config-dynamic.yaml COMPUTE {{ ydb_cores_dynamic | default(ydb_cores_dynamic, true) }}"
     changed_when: false
     when: not ydb_config_v2 | default(false)
     tags:

playbooks/update_executable.yaml

@@ -6,11 +6,11 @@
   tasks:
   - ansible.builtin.assert:
       that:
-        - "{{ item.var1 is defined or item.var2 is defined }}"
-      fail_msg: "Either {{ item.var1 }} or {{ item.var2 }} variable is required"
+        - "{{ item.var1 is defined"
+      fail_msg: "Either {{ item.var1 }} variable is required"
     loop:
-      - { var1: "ydb_storage_node_cores", var2: "ydb_cores_static" }
-      - { var1: "ydb_database_node_cores", var2: "ydb_cores_dynamic" }
+      - { var1: "ydb_cores_static" }
+      - { var1: "ydb_cores_dynamic" }
   tags:
     - storage
     - static

plugins/inventory/ydb_inventory.py

@@ -0,0 +1,122 @@
+from ansible.plugins.inventory import BaseInventoryPlugin
+import yaml
+import copy
+from ansible.errors import AnsibleError
+# from ansible.template import Templar
+
+DOCUMENTATION = r'''
+    name: ydb_inventory
+    plugin_type: inventory
+    short_description: YDB inventory from config.yaml
+    description: |
+        This inventory plugin fetches hosts from a custom source (config.yaml).
+    options:
+        plugin:
+            description: The name of the plugin (must be 'ydb_inventory')
+            required: true
+            choices: ['ydb_platform.ydb.ydb_inventory']
+        ydb_config:
+            description: YDB config (file or dictionary)
+            required: true
+            env:
+              - name: INVENTORY_YDB_CONFIG
+'''
+
+class InventoryModule(BaseInventoryPlugin):
+    NAME = 'ydb_inventory'
+
+    def verify_file(self, path):
+        # Проверяем, что это конфиг для нашего плагина
+        valid = super().verify_file(path)
+        return valid
+
+    def parse(self, inventory, loader, path, cache=True):
+        super().parse(inventory, loader, path, cache)
+        # Загружаем конфиг плагина (YAML-файл)
+        config = self._read_config_data(path)
+
+        ydb_config = config.get('ydb_config')
+        print(f"Reading inventory from {ydb_config}")
+
+        group = 'ydb'
+        self.inventory.add_group(group)
+        brokers = []
+
+        ydb_vars = self.inventory.groups['ydb'].get_vars()
+        try:
+            with open(ydb_config, "r") as file:
+                yaml_config = yaml.safe_load(file)
+
+                if 'config' in yaml_config:
+                    """ V2 Config """
+                    yaml_config = yaml_config['config']
+
+                self.inventory.groups['ydb'].set_variable('ydb_config_dict', yaml_config)
+
+                if 'ydb_dbname' not in ydb_vars and 'ydb_dynnodes' in ydb_vars:
+                    for dynnode in ydb_vars['ydb_dynnodes']:
+                        if 'dbname' in dynnode:
+                            self.inventory.groups['ydb'].set_variable('ydb_dbname',dynnode['dbname'])
+                            break
+
+                if 'ydb_enforce_user_token_requirement' not in ydb_vars:
+                    if 'domains_config' in yaml_config and 'security_config' in yaml_config['domains_config'] and 'enforce_user_token_requirement' in yaml_config['domains_config']['security_config']:
+                        self.inventory.groups['ydb'].set_variable('ydb_enforce_user_token_requirement', yaml_config['domains_config']['security_config']['enforce_user_token_requirement'])
+                    else:
+                        self.inventory.groups['ydb'].set_variable('ydb_enforce_user_token_requirement', False)
+
+                if 'ydb_pool_kind' not in ydb_vars:
+                    if 'default_disk_type' in yaml_config:
+                        self.inventory.groups['ydb'].set_variable('ydb_pool_kind', yaml_config['default_disk_type'].lower())
+                    if 'domains_config' in yaml_config and 'domain' in yaml_config['domains_config']:
+                        if 'storage_pool_types' in yaml_config['domains_config']['domain'][0]:
+                            self.inventory.groups['ydb'].set_variable('ydb_pool_kind', yaml_config['domains_config']['domain'][0]['storage_pool_types'][0]['kind'])
+
+                if 'self_management_config' in yaml_config and 'enabled' in yaml_config['self_management_config'] and yaml_config['self_management_config']['enabled']:
+                    self.inventory.groups['ydb'].set_variable('ydb_config_v2', True)
+
+                self.inventory.groups['ydb'].set_variable('ydb_config', yaml_config)
+
+                if 'ydb_domain' not in ydb_vars:
+                    if 'domains_config' in yaml_config and 'domain' in yaml_config['domains_config']:
+                        self.inventory.groups['ydb'].set_variable('ydb_domain',yaml_config['domains_config']['domain'][0]['name'])
+                    else:
+                        # Default domain is Root
+                        self.inventory.groups['ydb'].set_variable('ydb_domain','Root')
+
+                # Read drives config
+                drive_configs = {}
+                drive_labels = {}
+                if 'ydb_disks' in ydb_vars:
+                    for disk in ydb_vars['ydb_disks']:
+                        drive_labels[disk['label']] = disk['name']
+                if 'host_configs' in yaml_config:
+                    for drive_config in yaml_config['host_configs']:
+                        drive_configs[drive_config['host_config_id']] = copy.deepcopy(drive_config['drive'])
+                        for i, item in enumerate(drive_config['drive']):
+                            label = item['path'].split('/')[-1]
+                            drive_configs[drive_config['host_config_id']][i]['label'] = label
+                            if label in drive_labels:
+                                drive_configs[drive_config['host_config_id']][i]['name']  = drive_labels[label]
+                            else:
+                                raise AnsibleError(f"Config parsing error, unable to find disk for label: {label}")
+                # Read hosts and define variables for them
+                if 'hosts' in yaml_config:
+                    for host in yaml_config['hosts']:
+                        self.inventory.add_host(host['host'], group=group)
+                        # Set variables for hosts
+                        for key, value in host.items():
+                            if key == 'host_config_id':
+                                self.inventory.set_variable(host['host'], 'ydb_disks', drive_configs[value])
+                            else:
+                                self.inventory.set_variable(host['host'], key, value)
+                        if len(brokers) < 3:
+                            brokers.append(host['host'])
+
+                if 'ydb_brokers' not in ydb_vars and len(brokers) > 0:
+                    self.inventory.groups['ydb'].set_variable('ydb_brokers', brokers)
+
+        except Exception as e:
+            raise AnsibleError(f"Config parsing error: {str(e)}")
+
+

plugins/modules/apply_config_selectors.py

@@ -9,6 +9,14 @@
 from ansible_collections.ydb_platform.ydb.plugins.module_utils.yaml_utils import safe_dump
 
 
+DOCUMENTATION = r'''
+    name: apply_config_selectors
+    plugin_type: module
+    short_description: Apply changes for storage selectors
+    description: |
+        Apply changes for storage selectors
+'''
+
 def run_module():
     module_args = dict(
         config=dict(type='str', required=True),

plugins/modules/cluster_config.py

@@ -6,6 +6,13 @@
 from ansible_collections.ydb_platform.ydb.plugins.module_utils import cli
 from ansible_collections.ydb_platform.ydb.plugins.module_utils.yaml_utils import safe_dump
 
+DOCUMENTATION = r'''
+    name: cluster_config
+    plugin_type: module
+    short_description: Work with ydb cluster config
+    description: |
+        Work with ydb cluster config (it's needed for V2 cluster configuration)
+'''
 
 def fetch_cluster_config(ydb_cli, result):
     """Fetch current cluster configuration from YDB"""

plugins/modules/create_database.py

@@ -4,6 +4,13 @@
 
 STDOUT_SUCCESS = 'OK'
 STDOUT_ALREADY_EXISTS = 'ERROR: ALREADY_EXISTS'
+DOCUMENTATION = r'''
+    name: create_database
+    plugin_type: module
+    short_description: Create YDB database
+    description: |
+        Create YDB database
+'''
 
 
 def main():

plugins/modules/drive_prepare.py

@@ -4,6 +4,14 @@
 from ansible.module_utils.basic import AnsibleModule
 from ansible_collections.ydb_platform.ydb.plugins.module_utils import drive, cli
 
+DOCUMENTATION = r'''
+    name: drive_prepare
+    plugin_type: module
+    short_description: Oblitirate drives for YDB
+    description: |
+        Module for preparing hard drives for YDB, it's safe format of drive. Drives with YDB data will not be affected / changed.
+        Be aware: it won't format drives with some YDB data for your safety
+'''
 
 def main():
     argument_spec = dict(