update C APIs

VERSION

@@ -1 +1 @@
-2.0.9
+2.0.10

lib/NGT/Capi.cpp

@@ -40,7 +40,23 @@ static bool operate_error_string_(const std::stringstream &ss, NGTError error){
 NGTIndex ngt_open_index(const char *index_path, NGTError error) {
   try{
     std::string index_path_str(index_path);
-    NGT::Index *index = new NGT::Index(index_path_str);
+    auto readOnly = false;
+    NGT::Index *index = new NGT::Index(index_path_str, readOnly);
+    index->disableLog();
+    return static_cast<NGTIndex>(index);
+  }catch(std::exception &err){
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what();
+    operate_error_string_(ss, error);
+    return NULL;
+  }
+}
+
+NGTIndex ngt_open_index_as_read_only(const char *index_path, NGTError error) {
+  try{
+    std::string index_path_str(index_path);
+    auto readOnly = true;
+    NGT::Index *index = new NGT::Index(index_path_str, readOnly);
     index->disableLog();
     return static_cast<NGTIndex>(index);
   }catch(std::exception &err){

lib/NGT/Capi.h

@@ -59,6 +59,8 @@ typedef struct {
 
 NGTIndex ngt_open_index(const char *, NGTError);
 
+NGTIndex ngt_open_index_as_read_only(const char *, NGTError);
+
 NGTIndex ngt_create_graph_and_tree(const char *, NGTProperty, NGTError);
 
 NGTIndex ngt_create_graph_and_tree_in_memory(NGTProperty, NGTError);

lib/NGT/NGTQ/Capi.cpp

@@ -209,10 +209,10 @@ bool qbg_create(const char *indexPath, QBGConstructionParameters *parameters, NG
   return true;
 }
 
-QBGIndex qbg_open_index(const char *index_path, QBGError error) {
+QBGIndex qbg_open_index(const char *index_path, bool read_only, QBGError error) {
   try {
     std::string index_path_str(index_path);
-    auto *index = new QBG::Index(index_path_str, true);
+    auto *index = new QBG::Index(index_path_str, read_only);
     return static_cast<QBGIndex>(index);
   } catch(std::exception &err){
     std::stringstream ss;
@@ -270,7 +270,7 @@ void qbg_initialize_build_parameters(QBGBuildParameters *parameters) {
   parameters->number_of_second_clusters = 0;
   parameters->number_of_third_clusters = 0;
 
-  parameters->number_of_objects = 0;
+  parameters->number_of_objects = 1000;
   parameters->number_of_subvectors = 1;
   parameters->optimization_clustering_init_mode = static_cast<int>(NGT::Clustering::InitializationModeKmeansPlusPlus);
   parameters->rotation_iteration = 2000;
@@ -333,8 +333,8 @@ bool qbg_build_index(const char *index_path, QBGBuildParameters *parameters, QBG
   optimizer.silence = true;
 
   try {
-    bool random = true;
-    optimizer.optimize(index_path, random);
+    auto nthreads = omp_get_max_threads();
+    optimizer.optimize(index_path, nthreads);
   } catch (NGT::Exception &err) {
     std::stringstream ss;
     ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what();

lib/NGT/NGTQ/Capi.h

@@ -14,7 +14,6 @@
 // limitations under the License.
 //
 
-
 #pragma once
 
 #ifdef __cplusplus
@@ -112,7 +111,7 @@ extern "C" {
 
   bool qbg_create(const char *indexPath, QBGConstructionParameters *parameters, QBGError error);
 
-  QBGIndex qbg_open_index(const char *index_path, QBGError error);
+  QBGIndex qbg_open_index(const char *index_path, bool read_only, QBGError error);
 
   void qbg_close_index(QBGIndex index);
 

lib/NGT/NGTQ/Optimizer.cpp

@@ -233,8 +233,10 @@ void QBG::Optimizer::optimize(const std::string indexPath, size_t threadSize) {
     }
     std::cerr << "optimize: # of objects=" << numberOfObjects << std::endl;
     if (numberOfObjects == 0) {
-      numberOfObjects = index.getQuantizer().objectList.size() - 1;
+      numberOfObjects = 1000;
+      numberOfObjects = index.getQuantizer().objectList.size() - 1 < numberOfObjects ? index.getQuantizer().objectList.size() - 1 : numberOfObjects;
     }
+    std::cerr << "optimize: updated # of objects=" << numberOfObjects << std::endl;
     std::cerr << "optimize: # of clusters=" << index.getQuantizer().property.localCentroidLimit << ":" << numberOfClusters << std::endl;
     if (index.getQuantizer().property.localCentroidLimit == 0 && numberOfClusters == 0) {
       std::stringstream msg;

lib/NGT/NGTQ/QuantizedBlobGraph.h

@@ -374,9 +374,7 @@ namespace QBG {
     NGT::ObjectID append(std::vector<float> &object) {
       NGT::ObjectID id = getQuantizer().objectList.size();
       id = id == 0 ? 1 : id;
-      std::cerr << "ID=" << getQuantizer().objectList.size();
       getQuantizer().objectList.put(id, object, &getQuantizer().globalCodebookIndex.getObjectSpace());
-      std::cerr << ":" << getQuantizer().objectList.size() << std::endl;
       return id;
     }
 

lib/NGT/NGTQ/Quantizer.h

@@ -2615,20 +2615,16 @@ class QuantizerInstance : public Quantizer {
       if (ifs) {
 	std::cerr << "loading rotation..." << std::endl;
 	rotation.deserialize(ifs);
-      } else {
-	std::cerr << "Warning. Not found the rotation file. " << streamName << std::endl;
       }
     }
     {
 #ifdef NGTQG_ROTATION
       std::string rqcbName(rootDirectory + "/rqcb");
       ifstream irfs(rqcbName);
       if (!irfs) {
-	std::cerr << "Warning. Not found the rqcb file. " << rqcbName << std::endl;
 	std::string qcbName(rootDirectory + "/qcb");
 	ifstream ifs(qcbName);
 	if (!ifs) {
-	  std::cerr << "Warning. Not found the qcb file. " << qcbName << std::endl;
 	} else {
 	  std::cerr << "loading the global codebooks..." << std::endl;
 	  quantizationCodebook.deserialize(ifs, readOnly);

samples/CMakeLists.txt

@@ -7,4 +7,6 @@ if( ${UNIX} )
 	add_subdirectory("${PROJECT_SOURCE_DIR}/samples/cosine-float")
 	add_subdirectory("${PROJECT_SOURCE_DIR}/samples/jaccard-sparse")
 	add_subdirectory("${PROJECT_SOURCE_DIR}/samples/qg-l2-float")
+	add_subdirectory("${PROJECT_SOURCE_DIR}/samples/qg-capi")
+	add_subdirectory("${PROJECT_SOURCE_DIR}/samples/qbg-capi")
 endif()

samples/qbg-capi/CMakeLists.txt

@@ -0,0 +1,9 @@
+if( ${UNIX} )
+	include_directories("${PROJECT_BINARY_DIR}/lib")
+        include_directories("${PROJECT_SOURCE_DIR}/lib")
+        link_directories("${PROJECT_SOURCE_DIR}/lib/NGT")
+
+	add_executable(qbg-capi qbg-capi.cpp)
+	add_dependencies(qbg-capi ngt)
+	target_link_libraries(qbg-capi ngt pthread)
+endif()

samples/qbg-capi/qbg-capi.cpp

@@ -0,0 +1,138 @@
+
+#include	"NGT/Index.h"
+#include	"NGT/NGTQ/Capi.h"
+int
+main(int argc, char **argv)
+{
+  std::string indexPath  = "index";
+  std::string objectFile = "sift-128-euclidean.tsv";
+  std::string queryFile  = "query.tsv";
+
+  std::cerr << "run the following commands to prepare data for this sample program." << std::endl;
+  std::cerr << "  curl -L -O https://github.com/yahoojapan/NGT/raw/main/tests/datasets/ann-benchmarks/sift-128-euclidean.tsv" << std::endl;
+  std::cerr << "  curl -L -O https://github.com/yahoojapan/NGT/raw/main/tests/datasets/ann-benchmarks/sift-128-euclidean_query.tsv" << std::endl;
+  std::cerr << "  head -1 sift-128-euclidean_query.tsv > query.tsv" << std::endl;
+  std::cerr << std::endl;
+  std::cerr << "index path=" << indexPath << std::endl;
+  std::cerr << "object file=" << objectFile << std::endl;
+  std::cerr << "query file=" << queryFile << std::endl;
+  std::cerr << std::endl;
+
+  size_t dimension = 128;
+  NGTError err = ngt_create_error_object();
+
+  std::cerr << "create an empty index..." << std::endl;
+  QBGConstructionParameters constructionParameters;
+  qbg_initialize_construction_parameters(&constructionParameters);
+  constructionParameters.dimension = dimension;
+  constructionParameters.number_of_subvectors = 64;
+  constructionParameters.number_of_blobs = 0;
+  if (!qbg_create(indexPath.c_str(), &constructionParameters, err)) {
+    std::cerr << "Cannot create" << std::endl;
+    std::cerr << ngt_get_error_string(err) << std::endl;
+    return 1;
+  }
+
+  std::cerr << "append objects..." << std::endl;
+  auto index = qbg_open_index(indexPath.c_str(), false, err);
+  if (index == 0) {
+    std::cerr << "Cannot open" << std::endl;
+    std::cerr << ngt_get_error_string(err) << std::endl;
+    return 1;
+  }
+
+  try {
+    std::ifstream is(objectFile);
+    std::string	line;
+    while (getline(is, line)) {
+      std::vector<float> obj;
+      std::stringstream	linestream(line);
+      while (!linestream.eof()) {
+	float value;
+	linestream >> value;
+	if (linestream.fail()) {
+	  obj.clear();
+	  break;
+	}
+	obj.push_back(value);
+      }
+      if (obj.empty()) {
+	std::cerr << "An empty line or invalid value: " << line << std::endl;
+	return 1;
+      }
+      if (qbg_append_object(index, obj.data(), dimension, err) == 0) {
+	std::cerr << ngt_get_error_string(err) << std::endl;
+	return 1;
+      }
+    }
+  } catch (...) {
+    std::cerr << "Error" << std::endl;
+    return 1;
+  }
+
+  qbg_save_index(index, err);
+  qbg_close_index(index);
+
+  std::cerr << "building the index..." << std::endl;
+  QBGBuildParameters buildParameters;
+  qbg_initialize_build_parameters(&buildParameters);
+  buildParameters.number_of_objects = 500;		
+  auto status = qbg_build_index(indexPath.c_str(), &buildParameters, err);
+  if (!status) {
+    std::cerr << "Cannot build. " << ngt_get_error_string(err) << std::endl;
+    return 1;
+  }
+
+  index = qbg_open_index(indexPath.c_str(), true, err);
+  if (index == 0) {
+    std::cerr << "Cannot open. " << ngt_get_error_string(err) << std::endl;
+    return 1;
+  }
+
+  std::ifstream	is(queryFile);
+  if (!is) {
+    std::cerr << "Cannot open the specified file. " << queryFile << std::endl;
+    return 1;
+  }
+
+  std::string line;
+  float queryVector[dimension];
+  if (getline(is, line)) {
+    std::vector<double> queryObject;
+    {
+      std::vector<std::string> tokens;
+      NGT::Common::tokenize(line, tokens, " \t");
+      tokens.resize(dimension);
+      if (tokens.size() != dimension) {
+	std::cerr << "dimension of the query is invalid. dimesion=" << tokens.size() << ":" << dimension << std::endl;
+	return 1;
+      }
+      for (std::vector<std::string>::iterator ti = tokens.begin(); ti != tokens.end(); ++ti) {
+	queryVector[distance(tokens.begin(), ti)] = NGT::Common::strtod(*ti);
+      }
+    }
+    QBGObjectDistances result = ngt_create_empty_results(err);
+    QBGQuery query;
+    qbg_initialize_query(&query);
+    query.query = &queryVector[0];
+    std::cerr << "search the index for the specified query..." << std::endl;
+    auto status = qbg_search_index(index, query, result, err);
+    if (!status) {
+      std::cerr << "Cannot search. " << ngt_get_error_string(err) << std::endl;
+      return 1;
+    }
+    auto rsize = qbg_get_result_size(result, err);
+    std::cout << "Rank\tID\tDistance" << std::endl;
+    for (size_t i = 0; i < rsize; i++) {
+      NGTObjectDistance object = qbg_get_result(result, i, err);
+      std::cout << i + 1 << "\t" << object.id << "\t" << object.distance << std::endl;
+    }
+
+    qbg_destroy_results(result);
+  }
+
+  qbg_close_index(index);
+  ngt_destroy_error_object(err);
+
+  return 0;
+}

samples/qg-capi/CMakeLists.txt

@@ -0,0 +1,9 @@
+if( ${UNIX} )
+	include_directories("${PROJECT_BINARY_DIR}/lib")
+        include_directories("${PROJECT_SOURCE_DIR}/lib")
+        link_directories("${PROJECT_SOURCE_DIR}/lib/NGT")
+
+	add_executable(qg-capi qg-capi.cpp)
+	add_dependencies(qg-capi ngt)
+	target_link_libraries(qg-capi ngt pthread)
+endif()