Merge branch 'main' into topic/rm-ucx-mo

Signed-off-by: Raul Akhmetshin <[email protected]>

Author: rakhmets

benchmark/kvbench/commands/args.py

@@ -214,6 +214,11 @@ def nixl_bench_args(func):
         type=str,
         help="Comma-separated GPU CUDA device id to use for communication (only used with GPUNETIO backend)",
     )(func)
+    func = click.option(
+        "--gpunetio_oob_list",
+        type=str,
+        help="OOB network interface name for control path (only used with GPUNETIO backend)",
+    )(func)
     func = click.option(
         "--hf3fs_iopool_size",
         type=int,

benchmark/kvbench/commands/nixlbench.py

@@ -63,6 +63,7 @@ def __init__(
         benchmark_group="default",
         gds_mt_num_threads=1,
         gpunetio_device_list="0",
+        gpunetio_oob_list="",
         hf3fs_iopool_size=64,
         obj_access_key="",
         obj_secret_key="",
@@ -115,6 +116,7 @@ def __init__(
             worker_type (str, optional): Type of worker. Defaults to "nixl".
             gds_mt_num_threads (int, optional): Number of threads for GDS_MT plugin. Defaults to 1.
             gpunetio_device_list (str, optional): GPU device list for GPUNETIO plugin. Defaults to "0".
+            gpunetio_oob_list (str, optional): OOB network interface name for control path for GPUNETIO plugin. Defaults to "".
             hf3fs_iopool_size (int, optional): IO pool size for HF3FS plugin. Defaults to 64.
             obj_access_key (str, optional): Access key for OBJ/S3 plugin. Defaults to "".
             obj_secret_key (str, optional): Secret key for OBJ/S3 plugin. Defaults to "".
@@ -162,6 +164,7 @@ def __init__(
         self.worker_type = worker_type
         self.gds_mt_num_threads = gds_mt_num_threads
         self.gpunetio_device_list = gpunetio_device_list
+        self.gpunetio_oob_list = gpunetio_oob_list
         self.hf3fs_iopool_size = hf3fs_iopool_size
         self.obj_access_key = obj_access_key
         self.obj_secret_key = obj_secret_key
@@ -320,6 +323,7 @@ def _params(self):
             "worker_type": self.worker_type,
             "gds_mt_num_threads": self.gds_mt_num_threads,
             "gpunetio_device_list": self.gpunetio_device_list,
+            "gpunetio_oob_list": self.gpunetio_oob_list,
             "hf3fs_iopool_size": self.hf3fs_iopool_size,
             "obj_access_key": self.obj_access_key,
             "obj_secret_key": self.obj_secret_key,
@@ -379,6 +383,7 @@ def defaults():
             "benchmark_group": "default",
             "gds_mt_num_threads": 1,
             "gpunetio_device_list": "0",
+            "gpunetio_oob_list": "",
             "hf3fs_iopool_size": 64,
             "obj_access_key": "",
             "obj_secret_key": "",

benchmark/nixlbench/README.md

@@ -33,7 +33,7 @@ A comprehensive benchmarking tool for the NVIDIA Inference Xfer Library (NIXL) t
 ## Features
 
 - **Multiple Communication Backends**: UCX, GPUNETIO, Mooncake, Libfabric for network communication
-- **Storage Backend Support**: GDS, GDS_MT, POSIX, HF3FS, OBJ (S3) for storage operations
+- **Storage Backend Support**: GDS, GDS_MT, POSIX, HF3FS, OBJ (S3), GUSLI for storage operations
 - **Flexible Communication Patterns**:
   - **Pairwise**: Point-to-point communication between pairs
   - **Many-to-one**: Multiple initiators to single target
@@ -200,6 +200,7 @@ For development environments or when Docker is not available.
 - **DOCA**: NVIDIA DOCA SDK for GPUNetIO
 - **AWS SDK C++**: For S3 object storage backend
 - **GDS**: NVIDIA GPUDirect Storage
+- **GUSLI**: G3+ User Space Access Library for direct block device access
 - **NVSHMEM**: Required for NVSHMEM worker type
 - **hwloc**: Hardware locality detection (required for Libfabric only)
 
@@ -304,6 +305,21 @@ sudo dpkg -i doca-host_3.1.0-091000-25.07-ubuntu2404_amd64.deb
 sudo apt-get update && sudo apt-get install -y doca-sdk-gpunetio libdoca-sdk-gpunetio-dev
 ```
 
+**GUSLI (Optional - for GUSLI backend):**
+```bash
+# Clone and build GUSLI
+git clone https://github.com/nvidia/gusli.git
+cd gusli
+make all BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=0
+
+# Install library and headers
+sudo cp libgusli_clnt.so /usr/lib/
+sudo cp gusli_*.hpp /usr/include/
+sudo ldconfig
+```
+
+**Note**: GUSLI must be built before building NIXL. See [GUSLI Plugin README](../../src/plugins/gusli/README.md) for detailed installation and usage instructions.
+
 #### Python Environment Setup
 ```bash
 # Install uv (modern Python package manager)
@@ -404,7 +420,7 @@ sudo systemctl start etcd && sudo systemctl enable etcd
 ```
 --runtime_type NAME        # Type of runtime to use [ETCD] (default: ETCD)
 --worker_type NAME         # Worker to use to transfer data [nixl, nvshmem] (default: nixl)
---backend NAME             # Communication backend [UCX, GDS, GDS_MT, POSIX, GPUNETIO, Mooncake, HF3FS, OBJ] (default: UCX)
+--backend NAME             # Communication backend [UCX, GDS, GDS_MT, POSIX, GPUNETIO, Mooncake, HF3FS, OBJ, GUSLI] (default: UCX)
 --benchmark_group NAME     # Name of benchmark group for parallel runs (default: default)
 --etcd_endpoints URL       # ETCD server URL for coordination (default: http://localhost:2379)
 ```
@@ -486,13 +502,26 @@ sudo systemctl start etcd && sudo systemctl enable etcd
 --obj_req_checksum TYPE    # Required checksum for S3 backend [supported, required] (default: supported)
 ```
 
+**GUSLI Backend:**
+```
+--device_list LIST                     # Device specs in format 'id:type:path' (e.g., '11:F:./store0.bin,27:K:/dev/nvme0n1')
+                                       # Type: F (file), K (kernel device), N (networked server with t/u prefix)
+--gusli_client_name NAME               # Client identifier (default: NIXLBench)
+--gusli_max_simultaneous_requests NUM  # Concurrent request limit (default: 32)
+--gusli_device_security LIST           # Comma-separated security flags per device (e.g., 'sec=0x3,sec=0x71')
+--gusli_bdev_byte_offset BYTES         # Starting LBA offset in bytes (default: 1048576)
+--gusli_config_file CONTENT            # Custom config file content (auto-generated if not provided)
+
+Note: storage_enable_direct is automatically enabled for GUSLI backend
+```
+
 ### Using ETCD for Coordination
 
 NIXL Benchmark uses an ETCD key-value store for coordination between benchmark workers. This is useful in containerized or cloud-native environments.
 
 **ETCD Requirements:**
 - **Required**: Network backends (UCX, GPUNETIO, Mooncake, Libfabric) and multi-node setups
-- **Optional**: Storage backends (GDS, GDS_MT, POSIX, HF3FS, OBJ, S3) running as single instances
+- **Optional**: Storage backends (GDS, GDS_MT, POSIX, HF3FS, OBJ, GUSLI) running as single instances
 - **Required**: Storage backends when `--etcd_endpoints` is explicitly specified
 
 **For multi-node benchmarks:**
@@ -573,6 +602,69 @@ The workers automatically coordinate ranks through ETCD as they connect.
 ./nixlbench --backend POSIX --filepath /mnt/storage/testfile --posix_api_type URING --storage_enable_direct
 ```
 
+**GUSLI Backend (G3+ User Space Access Library):**
+
+GUSLI provides direct user-space access to block storage devices, supporting local files, kernel block devices, and networked GUSLI servers.
+
+**Note**: Direct I/O is automatically enabled when GUSLI backend is selected (no need to specify `--storage_enable_direct`).
+
+```bash
+# Basic GUSLI benchmark - single file device
+./nixlbench --backend=GUSLI \
+           --device_list="11:F:./store0.bin" \
+           --num_initiator_dev=1 \
+           --num_target_dev=1 \
+           --op_type=WRITE
+
+# NVMe device with custom security
+./nixlbench --backend=GUSLI \
+           --device_list="27:K:/dev/nvme0n1" \
+           --gusli_device_security="sec=0x7" \
+           --num_initiator_dev=1 \
+           --num_target_dev=1 \
+           --op_type=READ
+
+# Multi-device configuration
+./nixlbench --backend=GUSLI \
+           --device_list="11:F:./store0.bin,14:K:/dev/zero,27:K:/dev/nvme0n1" \
+           --gusli_device_security="sec=0x3,sec=0x71,sec=0x7" \
+           --num_initiator_dev=3 \
+           --num_target_dev=3 \
+           --op_type=WRITE
+
+# Networked GUSLI server (TCP)
+./nixlbench --backend=GUSLI \
+           --device_list="20:N:t192.168.1.100" \
+           --gusli_device_security="sec=0x10" \
+           --num_initiator_dev=1 \
+           --num_target_dev=1 \
+           --op_type=WRITE
+
+# High concurrency with multiple threads
+./nixlbench --backend=GUSLI \
+           --device_list="27:K:/dev/nvme0n1" \
+           --gusli_max_simultaneous_requests=128 \
+           --num_threads=8 \
+           --total_buffer_size=$((16*1024*1024*1024)) \
+           --op_type=WRITE
+```
+
+**GUSLI Device Types:**
+- `F`: File-backed storage (e.g., `11:F:./store0.bin`)
+- `K`: Kernel block device (e.g., `27:K:/dev/nvme0n1`, `14:K:/dev/zero`)
+- `N`: Networked GUSLI server with protocol prefix (e.g., `20:N:t192.168.1.100` for TCP, `21:N:u10.0.0.5` for UDP)
+
+**GUSLI-Specific Parameters:**
+- `--gusli_client_name`: Client identifier (default: "NIXLBench")
+- `--gusli_max_simultaneous_requests`: Concurrent request limit (default: 32)
+- `--gusli_device_security`: Comma-separated security flags per device (default: "sec=0x3" for each device)
+- `--gusli_bdev_byte_offset`: Starting LBA offset in bytes (default: 1MB)
+- `--gusli_config_file`: Custom config file content override
+
+**Notes**:
+- Number of devices in `--device_list` must match `--num_initiator_dev` and `--num_target_dev`
+- Direct I/O is automatically enabled for GUSLI (no need to specify `--storage_enable_direct`)
+
 ### Worker Types
 
 **NVSHMEM Worker:**

benchmark/nixlbench/src/utils/utils.cpp

@@ -50,9 +50,9 @@ DEFINE_string(
     "Name of NIXL backend [UCX, GDS, GDS_MT, POSIX, GPUNETIO, Mooncake, HF3FS, OBJ, GUSLI] \
               (only used with nixl worker)");
 DEFINE_string(initiator_seg_type, XFERBENCH_SEG_TYPE_DRAM, "Type of memory segment for initiator \
-              [DRAM, VRAM, BLK]");
+              [DRAM, VRAM]. Note: Storage backends always use DRAM locally.");
 DEFINE_string(target_seg_type, XFERBENCH_SEG_TYPE_DRAM, "Type of memory segment for target \
-              [DRAM, VRAM, BLK]");
+              [DRAM, VRAM]. Note: Storage backends determine remote type automatically.");
 DEFINE_string(scheme, XFERBENCH_SCHEME_PAIRWISE, "Scheme: pairwise, maytoone, onetomany, tp");
 DEFINE_string(mode, XFERBENCH_MODE_SG, "MODE: SG (Single GPU per proc), MG (Multi GPU per proc) [default: SG]");
 DEFINE_string(op_type, XFERBENCH_OP_WRITE, "Op type: READ, WRITE");
@@ -109,6 +109,9 @@ DEFINE_string (posix_api_type,
 // DOCA GPUNetIO options - only used when backend is DOCA GPUNetIO
 DEFINE_string(gpunetio_device_list, "0", "Comma-separated GPU CUDA device id to use for \
 		      communication (only used with nixl worker)");
+// DOCA GPUNetIO options - only used when backend is DOCA GPUNetIO
+DEFINE_string(gpunetio_oob_list, "", "Comma-separated OOB network interface name \
+		      for control path (only used with nixl worker)");
 
 // OBJ options - only used when backend is OBJ
 DEFINE_string(obj_access_key, "", "Access key for S3 backend");
@@ -132,12 +135,19 @@ DEFINE_string(gusli_client_name, "NIXLBench", "Client name for GUSLI backend");
 DEFINE_int32(gusli_max_simultaneous_requests,
              32,
              "Maximum number of simultaneous requests for GUSLI backend");
-DEFINE_string(gusli_config_file,
-              "",
-              "Configuration file content for GUSLI backend (if empty, uses default config)");
+DEFINE_string(
+    gusli_config_file,
+    "",
+    "Configuration file content for GUSLI backend (if empty, auto-generated from device_list)");
 DEFINE_uint64(gusli_bdev_byte_offset,
               1048576,
               "Byte offset in block device for GUSLI operations (default: 1MB)");
+DEFINE_string(gusli_device_security,
+              "",
+              "Comma-separated list of security flags per device (e.g. 'sec=0x3,sec=0x71'). "
+              "If empty or fewer than devices, uses 'sec=0x3' as default. "
+              "For GUSLI backend, use device_list in format 'id:type:path' where type is F (file) "
+              "or K (kernel device).");
 
 std::string xferBenchConfig::runtime_type = "";
 std::string xferBenchConfig::worker_type = "";
@@ -169,6 +179,7 @@ int xferBenchConfig::gds_batch_pool_size = 0;
 int xferBenchConfig::gds_batch_limit = 0;
 int xferBenchConfig::gds_mt_num_threads = 0;
 std::string xferBenchConfig::gpunetio_device_list = "";
+std::string xferBenchConfig::gpunetio_oob_list = "";
 std::vector<std::string> devices = { };
 int xferBenchConfig::num_files = 0;
 std::string xferBenchConfig::posix_api_type = "";
@@ -190,6 +201,7 @@ std::string xferBenchConfig::gusli_client_name = "";
 int xferBenchConfig::gusli_max_simultaneous_requests = 0;
 std::string xferBenchConfig::gusli_config_file = "";
 uint64_t xferBenchConfig::gusli_bdev_byte_offset = 0;
+std::string xferBenchConfig::gusli_device_security = "";
 
 int
 xferBenchConfig::loadFromFlags() {
@@ -237,6 +249,7 @@ xferBenchConfig::loadFromFlags() {
         // Load DOCA-specific configurations if backend is DOCA
         if (backend == XFERBENCH_BACKEND_GPUNETIO) {
             gpunetio_device_list = FLAGS_gpunetio_device_list;
+            gpunetio_oob_list = FLAGS_gpunetio_oob_list;
         }
 
         // Load HD3FS-specific configurations if backend is HD3FS
@@ -250,6 +263,7 @@ xferBenchConfig::loadFromFlags() {
             gusli_max_simultaneous_requests = FLAGS_gusli_max_simultaneous_requests;
             gusli_config_file = FLAGS_gusli_config_file;
             gusli_bdev_byte_offset = FLAGS_gusli_bdev_byte_offset;
+            gusli_device_security = FLAGS_gusli_device_security;
         }
 
         // Load OBJ-specific configurations if backend is OBJ
@@ -466,6 +480,8 @@ xferBenchConfig::printConfig() {
         if (backend == XFERBENCH_BACKEND_GPUNETIO) {
             printOption ("GPU CUDA Device id list (--device_list=dev1,dev2,...)",
                          gpunetio_device_list);
+            printOption("OOB network interface name for control path (--oob_list=ifface)",
+                        gpunetio_oob_list);
         }
     }
     printOption ("Initiator seg type (--initiator_seg_type=[DRAM,VRAM])", initiator_seg_type);

benchmark/nixlbench/src/utils/utils.h

@@ -157,6 +157,7 @@ class xferBenchConfig {
         static int gds_batch_limit;
         static int gds_mt_num_threads;
         static std::string gpunetio_device_list;
+        static std::string gpunetio_oob_list;
         static long page_size;
         static std::string obj_access_key;
         static std::string obj_secret_key;
@@ -173,6 +174,7 @@ class xferBenchConfig {
         static int gusli_max_simultaneous_requests;
         static std::string gusli_config_file;
         static uint64_t gusli_bdev_byte_offset;
+        static std::string gusli_device_security;
 
         static int
         loadFromFlags();