Add env param to kikimr_configure --enable_modules, default false (#27724)

Co-authored-by: Alexander Yakovlev <[email protected]>

Author: AleYakovlev

ydb/tools/cfg/base.py

@@ -105,7 +105,7 @@ def __hash__(self):
 )
 
 KiKiMRHost = collections.namedtuple(
-    "_KiKiMRHost", ["hostname", "node_id", "drives", "ic_port", "body", "datacenter", "rack", "host_config_id", "port"]
+    "_KiKiMRHost", ["hostname", "node_id", "drives", "ic_port", "body", "datacenter", "module", "rack", "host_config_id", "port"]
 )
 
 DEFAULT_PLAN_RESOLUTION = 10
@@ -417,6 +417,16 @@ def _get_body(self, host_description):
 
         return str(self._host_info_provider.get_body(host_description.get("name", host_description.get("host"))))
 
+    def _get_module(self, host_description):
+        if host_description.get("module") is not None:
+            return str(host_description.get("module"))
+        module = host_description.get("location", {}).get("module", None)
+        if module is not None:
+            return str(module)
+
+        module_from_provider = self._host_info_provider.get_module(host_description.get("name", host_description.get("host")))
+        return str(module_from_provider) if module_from_provider else ""
+
     def _collect_drives_info(self, host_description):
         host_config_id = host_description.get("host_config_id", None)
         drives = host_description.get("drives", [])
@@ -439,6 +449,7 @@ def __collect_host_info(self, node_id, host_description):
             ic_port=host_description.get("ic_port", DEFAULT_INTERCONNECT_PORT),
             body=self._get_body(host_description),
             datacenter=self._get_datacenter(host_description),
+            module=self._get_module(host_description),
             rack=self._get_rack(host_description),
             host_config_id=host_description.get("host_config_id", None),
             port=host_description.get("port", DEFAULT_INTERCONNECT_PORT),

ydb/tools/cfg/configurator_setup.py

@@ -22,6 +22,7 @@ def parse_optional_arguments(args):
         kwargs['walle_url'] = args.walle_url
 
     kwargs['enable_cores'] = args.enable_cores
+    kwargs['enable_modules'] = getattr(args, 'enable_modules', False)
 
     for cmd_arg in ('ic_port', 'grpc_port', 'mbus_port', 'mon_port', 'cfg_home', 'sqs_port', 'binaries_home'):
         if hasattr(args, cmd_arg) and getattr(args, cmd_arg) is not None:
@@ -59,6 +60,9 @@ def get_parser(generate_func, extra_cfg_arguments=[]):
         '--tenant', type=str, help='Tenant to serve (possible options: name of domain or no for pure storage nodes)'
     )
     parser_cfg.add_argument('--enable-cores', action='store_true', help='Enables coredumps')
+    parser_cfg.add_argument(
+        '--enable-modules', action='store_true', help='Enable module field in node location configuration (default: false)'
+    )
     parser_cfg.add_argument(
         '--dynamic-node', action='store_true', help='Indicates that configuration should be generated for dynamic node'
     )

ydb/tools/cfg/dynamic.py

@@ -32,7 +32,7 @@ def __init__(
         self._grpc_endpoint = grpc_endpoint
         self.__configure_request = None
         self.__static_config = static.StaticConfigGenerator(
-            template, binary_path, output_dir, host_info_provider=host_info_provider, local_binary_path=local_binary_path
+            template, binary_path, output_dir, host_info_provider=host_info_provider, local_binary_path=local_binary_path, **kwargs
         )
 
     @property

ydb/tools/cfg/static.py

@@ -53,6 +53,7 @@ def __init__(
         cfg_home="/Berkanavt/kikimr",
         sqs_port=8771,
         enable_cores=False,
+        enable_modules=False,
         local_binary_path=None,
         skip_location=False,
         schema_validator=None,
@@ -70,6 +71,7 @@ def __init__(
             self._host_info_provider._init_k8s_labels(self.__cluster_details.k8s_rack_label, self.__cluster_details.k8s_dc_label)
 
         self._enable_cores = template.get("enable_cores", enable_cores)
+        self._enable_modules = template.get("enable_modules", enable_modules)
         self._yaml_config_enabled = template.get("yaml_config_enabled", False)
         self.__is_dynamic_node = True if database is not None else False
         self._database = database
@@ -1509,6 +1511,8 @@ def __generate_names_txt(self):
             if not self._skip_location:
                 if self.__cluster_details.use_walle:
                     node.WalleLocation.DataCenter = host.datacenter
+                    if self._enable_modules:
+                        node.WalleLocation.Module = host.module
                     node.WalleLocation.Rack = host.rack
                     node.WalleLocation.Body = int(host.body)
                 elif self.__cluster_details.use_k8s_api:
@@ -1517,6 +1521,8 @@ def __generate_names_txt(self):
                     node.Location.Body = int(host.body)
                 else:
                     node.Location.DataCenter = host.datacenter
+                    if self._enable_modules:
+                        node.Location.Module = host.module
                     node.Location.Rack = host.rack
                     node.Location.Body = int(host.body)
 

ydb/tools/cfg/walle/walle.py

@@ -17,15 +17,19 @@ class HostsInformationProvider:
     __metaclass__ = ABCMeta
 
     @abstractmethod
-    def get_rack(hostname):
+    def get_rack(self, hostname):
         pass
 
     @abstractmethod
-    def get_datacenter(hostname):
+    def get_datacenter(self, hostname):
         pass
 
     @abstractmethod
-    def get_body(hostname):
+    def get_body(self, hostname):
+        pass
+
+    @abstractmethod
+    def get_module(self, hostname):
         pass
 
 
@@ -44,6 +48,9 @@ def get_datacenter(self, hostname):
     def get_body(self, hostname):
         return zlib.crc32(hostname.encode())
 
+    def get_module(self, hostname):
+        return "FAKE"
+
 
 class WalleHostsInformationProvider(HostsInformationProvider):
     def __init__(self, provider_url=None, cloud_mode=False):
@@ -103,5 +110,9 @@ def get_datacenter(self, hostname):
         short_dc_name = self._ask_location(hostname)["location"]["short_datacenter_name"]
         return short_dc_name if self._cloud_mode else short_dc_name.upper()
 
+    def get_module(self, hostname):
+        queue = self._ask_location(hostname)["location"].get("queue", "")
+        return queue if self._cloud_mode else queue.upper()
+
     def get_body(self, hostname):
         return self._ask_location(hostname)["inv"]