Merge remote-tracking branch 'upstream/main' into HH-migrate-merge-dataset-to-take-inputs-concat-buffer-and-offset-26_10

Author: HowardHuang1

c/include/cuvs/neighbors/cagra.h

@@ -168,7 +168,10 @@ struct cuvsAceParams {
    *
    * Used when `use_disk` is true or when the graph does not fit in host and GPU
    * memory. This should be the fastest disk in the system and hold enough space
-   * for twice the dataset, final graph, and label mapping.
+   * for twice the dataset, final graph, and label mapping. The directory may
+   * already exist, but ACE's named artifacts must not already exist. Simultaneous
+   * builds must use different directories. On failure, ACE removes only artifacts
+   * it created and never deletes unrelated directory contents.
    */
   const char* build_dir;
   /**

c/include/cuvs/neighbors/hnsw.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION.
+ * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  */
 
@@ -66,7 +66,11 @@ struct cuvsHnswAceParams {
   size_t npartitions;
   /**
    * Directory to store ACE build artifacts (e.g., KNN graph, optimized graph).
-   * Used when `use_disk` is true or when the graph does not fit in memory.
+   * Used when `use_disk` is true or when the graph does not fit in memory. The
+   * directory may already exist, but ACE's named artifacts and `hnsw_index.bin`
+   * must not already exist. Simultaneous builds must use different directories.
+   * On failure, ACE removes only its uncommitted CAGRA artifacts; a completed
+   * CAGRA stage is retained if creating the HNSW index fails.
    */
   const char* build_dir;
   /**
@@ -109,7 +113,7 @@ CUVS_EXPORT cuvsError_t cuvsHnswAceParamsDestroy(cuvsHnswAceParams_t params);
 struct cuvsHnswIndexParams {
   /* hierarchy of the hnsw index */
   enum cuvsHnswHierarchy hierarchy;
-  /** Size of the candidate list during hierarchy construction when hierarchy is `CPU`*/
+  /** Maximum candidate list size used during index construction. */
   int ef_construction;
   /** Number of host threads to use to construct hierarchy when hierarchy is `CPU` or `GPU`.
       When the value is 0, the number of threads is automatically determined to the
@@ -119,15 +123,15 @@ struct cuvsHnswIndexParams {
       is parallelized with the help of CPU threads.
   */
   int num_threads;
-  /** HNSW M parameter: number of bi-directional links per node (used when building with ACE).
-   *  graph_degree = m * 2, intermediate_graph_degree = m * 3.
+  /** HNSW M parameter: number of bi-directional links per node. When the graph is built on the GPU,
+   *  this parameter is used to derive the internal CAGRA graph build parameters.
    */
   size_t M;
   /** Distance type for the index. */
   cuvsDistanceType metric;
   /**
-   * Optional: specify ACE parameters for building HNSW index using ACE algorithm.
-   * Set to nullptr for default behavior (from_cagra conversion).
+   * Optional ACE parameters for out-of-core graph construction.
+   * Set to nullptr to select the graph build algorithm automatically.
    */
   cuvsHnswAceParams_t ace_params;
 };
@@ -285,22 +289,20 @@ CUVS_EXPORT cuvsError_t cuvsHnswFromCagraWithDataset(cuvsResources_t res,
  */
 
 /**
- * @defgroup hnsw_c_index_build Build HNSW index using ACE algorithm
+ * @defgroup hnsw_c_index_build Build an HNSW index
  * @{
  */
 
 /**
- * @brief Build an HNSW index using ACE (Augmented Core Extraction) algorithm.
+ * @brief Build an HNSW index from HNSW parameters.
  *
- * ACE enables building HNSW indexes for datasets too large to fit in GPU memory by:
- * 1. Partitioning the dataset using balanced k-means into core and augmented partitions
- * 2. Building sub-indexes for each partition independently
- * 3. Concatenating sub-graphs into a final unified index
+ * The graph is built on the GPU and converted to an HNSW index that can be searched on the CPU.
+ * The graph build algorithm is selected automatically unless explicit ACE parameters are provided.
  *
  * NOTE: This function requires CUDA to be available at runtime.
  *
  * @param[in] res cuvsResources_t opaque C handle
- * @param[in] params cuvsHnswIndexParams_t with ACE parameters configured
+ * @param[in] params cuvsHnswIndexParams_t with HNSW build parameters
  * @param[in] dataset DLManagedTensor* host dataset to build index from
  * @param[out] index cuvsHnswIndex_t to return the built HNSW index
  *
@@ -314,18 +316,10 @@ CUVS_EXPORT cuvsError_t cuvsHnswFromCagraWithDataset(cuvsResources_t res,
  * cuvsResources_t res;
  * cuvsResourcesCreate(&res);
  *
- * // Create ACE parameters
- * cuvsHnswAceParams_t ace_params;
- * cuvsHnswAceParamsCreate(&ace_params);
- * ace_params->npartitions = 4;
- * ace_params->use_disk = true;
- * ace_params->build_dir = "/tmp/hnsw_ace_build";
- *
  * // Create index parameters
  * cuvsHnswIndexParams_t params;
  * cuvsHnswIndexParamsCreate(&params);
  * params->hierarchy = GPU;
- * params->ace_params = ace_params;
  * params->M = 32;
  * params->ef_construction = 120;
  *
@@ -340,7 +334,6 @@ CUVS_EXPORT cuvsError_t cuvsHnswFromCagraWithDataset(cuvsResources_t res,
  * cuvsHnswBuild(res, params, &dataset, hnsw_index);
  *
  * // Clean up
- * cuvsHnswAceParamsDestroy(ace_params);
  * cuvsHnswIndexParamsDestroy(params);
  * cuvsHnswIndexDestroy(hnsw_index);
  * cuvsResourcesDestroy(res);

c/src/neighbors/hnsw.cpp

@@ -38,15 +38,17 @@ void _build(cuvsResources_t res,
   cpp_params.M               = params->M;
   cpp_params.metric          = static_cast<cuvs::distance::DistanceType>(params->metric);
 
-  // Configure ACE parameters
-  RAFT_EXPECTS(params->ace_params != nullptr, "ACE parameters must be set for hnsw::build");
-  auto ace_params                 = cuvs::neighbors::hnsw::graph_build_params::ace_params();
-  ace_params.npartitions          = params->ace_params->npartitions;
-  ace_params.build_dir            = params->ace_params->build_dir ? params->ace_params->build_dir : "/tmp/hnsw_ace_build";
-  ace_params.use_disk             = params->ace_params->use_disk;
-  ace_params.max_host_memory_gb   = params->ace_params->max_host_memory_gb;
-  ace_params.max_gpu_memory_gb    = params->ace_params->max_gpu_memory_gb;
-  cpp_params.graph_build_params   = ace_params;
+  if (params->ace_params != nullptr) {
+    auto ace_params               = cuvs::neighbors::hnsw::graph_build_params::ace_params();
+    ace_params.npartitions        = params->ace_params->npartitions;
+    ace_params.build_dir          = params->ace_params->build_dir
+                                      ? params->ace_params->build_dir
+                                      : "/tmp/hnsw_ace_build";
+    ace_params.use_disk           = params->ace_params->use_disk;
+    ace_params.max_host_memory_gb = params->ace_params->max_host_memory_gb;
+    ace_params.max_gpu_memory_gb  = params->ace_params->max_gpu_memory_gb;
+    cpp_params.graph_build_params = ace_params;
+  }
 
   using dataset_mdspan_type = raft::host_matrix_view<T const, int64_t, raft::row_major>;
   auto dataset_mds          = cuvs::core::from_dlpack<dataset_mdspan_type>(dataset_tensor);

c/tests/neighbors/ann_hnsw_c.cu

@@ -136,6 +136,37 @@ TEST(CagraHnswC, BuildSearch)
   cuvsResourcesDestroy(res);
 }
 
+TEST(HnswC, BuildWithoutAce)
+{
+  cuvsResources_t res;
+  cuvsResourcesCreate(&res);
+
+  DLManagedTensor dataset_tensor;
+  dataset_tensor.dl_tensor.data               = dataset;
+  dataset_tensor.dl_tensor.device.device_type = kDLCPU;
+  dataset_tensor.dl_tensor.ndim               = 2;
+  dataset_tensor.dl_tensor.dtype.code         = kDLFloat;
+  dataset_tensor.dl_tensor.dtype.bits         = 32;
+  dataset_tensor.dl_tensor.dtype.lanes        = 1;
+  int64_t dataset_shape[2]                    = {4, 2};
+  dataset_tensor.dl_tensor.shape              = dataset_shape;
+  dataset_tensor.dl_tensor.strides            = nullptr;
+
+  cuvsHnswIndexParams_t hnsw_params;
+  cuvsHnswIndexParamsCreate(&hnsw_params);
+  hnsw_params->M               = 2;
+  hnsw_params->ef_construction = 100;
+
+  cuvsHnswIndex_t hnsw_index;
+  cuvsHnswIndexCreate(&hnsw_index);
+
+  ASSERT_EQ(cuvsHnswBuild(res, hnsw_params, &dataset_tensor, hnsw_index), CUVS_SUCCESS);
+
+  cuvsHnswIndexParamsDestroy(hnsw_params);
+  cuvsHnswIndexDestroy(hnsw_index);
+  cuvsResourcesDestroy(res);
+}
+
 TEST(HnswAceC, BuildSearch)
 {
   // create cuvsResources_t

cpp/include/cuvs/neighbors/cagra.hpp

@@ -67,7 +67,10 @@ struct ace_params {
    *
    * Used when `use_disk` is true or when the graph does not fit in host and GPU
    * memory. This should be the fastest disk in the system and hold enough space
-   * for twice the dataset, final graph, and label mapping.
+   * for twice the dataset, final graph, and label mapping. The directory may
+   * already exist, but ACE's named artifacts must not already exist. Simultaneous
+   * builds must use different directories. On failure, ACE removes only artifacts
+   * it created and never deletes unrelated directory contents.
    */
   std::string build_dir = "/tmp/ace_build";
   /**

cpp/include/cuvs/neighbors/hnsw.hpp

@@ -79,6 +79,13 @@ struct index_params : cuvs::neighbors::index_params {
    * ace.use_disk = true;
    * ace.build_dir = "/tmp/hnsw_ace_build";
    * @endcode
+   *
+   * When ACE writes to disk, `build_dir` may already exist, but ACE's named CAGRA
+   * artifacts and `hnsw_index.bin` must not already exist. Simultaneous builds
+   * must use different directories. The HNSW output is published only after it
+   * is fully serialized; however, the complete build is not transactional: if
+   * HNSW conversion fails after CAGRA succeeds, the completed CAGRA artifacts
+   * remain in the directory.
    */
   std::variant<std::monostate, graph_build_params::ace_params> graph_build_params;
 };

cpp/include/cuvs/util/file_io.hpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION.
+ * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  */
 #pragma once
@@ -180,47 +180,58 @@ class file_descriptor {
  * @tparam T Data type for the numpy array
  * @param path File path to create
  * @param shape Shape of the numpy array (e.g., {rows, cols} for 2D)
+ * @param exclusive Fail when the file already exists instead of truncating it.
+ *                  If creation succeeds but pre-allocation or header writing
+ *                  fails, remove the newly created file.
  * @return Pair of (file_descriptor, header_size)
  */
 template <typename T>
 std::pair<file_descriptor, size_t> create_numpy_file(const std::string& path,
-                                                     const std::vector<size_t>& shape)
+                                                     const std::vector<size_t>& shape,
+                                                     bool exclusive = false)
 {
   // Open file
-  file_descriptor fd(path, O_CREAT | O_RDWR | O_TRUNC, 0644);
-
-  // Build header
-  const auto dtype                              = raft::numpy_serializer::get_numpy_dtype<T>();
-  const bool fortran_order                      = false;
-  const raft::numpy_serializer::header_t header = {dtype, fortran_order, shape};
-
-  std::stringstream ss;
-  raft::numpy_serializer::write_header(ss, header);
-  std::string header_str = ss.str();
-  size_t header_size     = header_str.size();
-
-  // Calculate data size from shape
-  size_t data_bytes = sizeof(T);
-  for (auto dim : shape) {
-    data_bytes *= dim;
-  }
+  const int flags = O_CREAT | O_RDWR | (exclusive ? O_EXCL : O_TRUNC);
+  file_descriptor fd(path, flags, 0644);
+
+  try {
+    // Build header
+    const auto dtype                              = raft::numpy_serializer::get_numpy_dtype<T>();
+    const bool fortran_order                      = false;
+    const raft::numpy_serializer::header_t header = {dtype, fortran_order, shape};
+
+    std::stringstream ss;
+    raft::numpy_serializer::write_header(ss, header);
+    std::string header_str = ss.str();
+    size_t header_size     = header_str.size();
+
+    // Calculate data size from shape
+    size_t data_bytes = sizeof(T);
+    for (auto dim : shape) {
+      data_bytes *= dim;
+    }
 
-  // Pre-allocate file space
-  if (posix_fallocate(fd.get(), 0, header_size + data_bytes) != 0) {
-    RAFT_FAIL("Failed to pre-allocate space for file: %s", path.c_str());
-  }
+    // Pre-allocate file space
+    if (posix_fallocate(fd.get(), 0, header_size + data_bytes) != 0) {
+      RAFT_FAIL("Failed to pre-allocate space for file: %s", path.c_str());
+    }
 
-  // Seek to beginning and write header
-  if (lseek(fd.get(), 0, SEEK_SET) == -1) {
-    RAFT_FAIL("Failed to seek to beginning of file: %s", path.c_str());
-  }
+    // Seek to beginning and write header
+    if (lseek(fd.get(), 0, SEEK_SET) == -1) {
+      RAFT_FAIL("Failed to seek to beginning of file: %s", path.c_str());
+    }
 
-  ssize_t written = write(fd.get(), header_str.data(), header_str.size());
-  if (written < 0 || static_cast<size_t>(written) != header_str.size()) {
-    RAFT_FAIL("Failed to write numpy header to file: %s", path.c_str());
-  }
+    ssize_t written = write(fd.get(), header_str.data(), header_str.size());
+    if (written < 0 || static_cast<size_t>(written) != header_str.size()) {
+      RAFT_FAIL("Failed to write numpy header to file: %s", path.c_str());
+    }
 
-  return {std::move(fd), header_size};
+    return {std::move(fd), header_size};
+  } catch (...) {
+    fd.close();
+    if (exclusive) { (void)::unlink(path.c_str()); }
+    throw;
+  }
 }
 
 /**