update C APIs

CMakeLists.txt

@@ -1,7 +1,7 @@
 if(APPLE)
-  cmake_minimum_required(VERSION 3.0)
+    cmake_minimum_required(VERSION 3.0)
 else()
-  cmake_minimum_required(VERSION 2.8)
+    cmake_minimum_required(VERSION 2.8)
 endif()
 
 project(ngt)
@@ -13,13 +13,17 @@ string(REGEX MATCH "^[0-9]+" ngt_VERSION_MAJOR ${ngt_VERSION})
 set(ngt_VERSION ${ngt_VERSION})
 set(ngt_SOVERSION ${ngt_VERSION_MAJOR})
 
-if (NOT CMAKE_BUILD_TYPE)
-	set (CMAKE_BUILD_TYPE "Release")
-endif (NOT CMAKE_BUILD_TYPE)
+if(NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE "Release")
+endif(NOT CMAKE_BUILD_TYPE)
 string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER)
 message(STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
 message(STATUS "CMAKE_BUILD_TYPE_LOWER: ${CMAKE_BUILD_TYPE_LOWER}")
 
+if(DEFINED NGT_SHARED_MEMORY_ALLOCATOR)
+    set(NGT_QBG_DISABLED TRUE)
+endif(DEFINED NGT_SHARED_MEMORY_ALLOCATOR)
+
 if(${UNIX})
     set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
 
@@ -93,4 +97,4 @@ if(${UNIX})
     add_subdirectory("${PROJECT_SOURCE_DIR}/lib")
     add_subdirectory("${PROJECT_SOURCE_DIR}/bin")
     add_subdirectory("${PROJECT_SOURCE_DIR}/samples")
-endif( ${UNIX} )
+endif(${UNIX})

VERSION

@@ -1 +1 @@
-2.0.11
+2.0.12

lib/NGT/Capi.cpp

@@ -737,6 +737,88 @@ ObjectID ngt_append_index_as_float(NGTIndex index, float *obj, uint32_t obj_dim,
   }
 }
 
+ObjectID ngt_insert_index_as_uint8(NGTIndex index, uint8_t *obj, uint32_t obj_dim, NGTError error) {
+  if(index == NULL || obj == NULL || obj_dim == 0){
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : parametor error: index = " << index << " obj = " << obj << " obj_dim = " << obj_dim;
+    operate_error_string_(ss, error);
+    return 0;
+  }
+
+  try{
+    NGT::Index* pindex = static_cast<NGT::Index*>(index);
+    std::vector<uint8_t> vobj(&obj[0], &obj[obj_dim]);
+    return pindex->insert(vobj);
+  }catch(std::exception &err) {
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what();
+    operate_error_string_(ss, error);
+    return 0;
+  }
+}
+
+ObjectID ngt_append_index_as_uint8(NGTIndex index, uint8_t *obj, uint32_t obj_dim, NGTError error) {
+  if(index == NULL || obj == NULL || obj_dim == 0){
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : parametor error: index = " << index << " obj = " << obj << " obj_dim = " << obj_dim;
+    operate_error_string_(ss, error);
+    return 0;
+  }
+
+  try{
+    NGT::Index* pindex = static_cast<NGT::Index*>(index);
+    std::vector<uint8_t> vobj(&obj[0], &obj[obj_dim]);
+    return pindex->append(vobj);
+  }catch(std::exception &err) {
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what();
+    operate_error_string_(ss, error);
+    return 0;
+  }
+}
+
+ObjectID ngt_insert_index_as_float16(NGTIndex index, NGTFloat16 *obj, uint32_t obj_dim, NGTError error) {
+  if(index == NULL || obj == NULL || obj_dim == 0){
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : parametor error: index = " << index << " obj = " << obj << " obj_dim = " << obj_dim;
+    operate_error_string_(ss, error);
+    return 0;
+  }
+
+  try{
+    NGT::Index* pindex = static_cast<NGT::Index*>(index);
+    auto o = static_cast<NGT::float16*>(obj);
+    std::vector<NGT::float16> vobj(&o[0], &o[obj_dim]);
+    return pindex->insert(vobj);
+  }catch(std::exception &err) {
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what();
+    operate_error_string_(ss, error);
+    return 0;
+  }
+}
+
+ObjectID ngt_append_index_as_float16(NGTIndex index, NGTFloat16 *obj, uint32_t obj_dim, NGTError error) {
+  if(index == NULL || obj == NULL || obj_dim == 0){
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : parametor error: index = " << index << " obj = " << obj << " obj_dim = " << obj_dim;
+    operate_error_string_(ss, error);
+    return 0;
+  }
+
+  try{
+    NGT::Index* pindex = static_cast<NGT::Index*>(index);
+    auto o = static_cast<NGT::float16*>(obj);
+    std::vector<NGT::float16> vobj(&o[0], &o[obj_dim]);
+    return pindex->append(vobj);
+  }catch(std::exception &err) {
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what();
+    operate_error_string_(ss, error);
+    return 0;
+  }
+}
+
 bool ngt_batch_append_index(NGTIndex index, float *obj, uint32_t data_count, NGTError error) {
   try{
     NGT::Index* pindex = static_cast<NGT::Index*>(index);
@@ -845,13 +927,64 @@ void* ngt_get_object(NGTObjectSpace object_space, ObjectID id, NGTError error) {
 }
 
 float* ngt_get_object_as_float(NGTObjectSpace object_space, ObjectID id, NGTError error) {
+  auto os = static_cast<NGT::ObjectSpace*>(object_space);
+  if (os->getObjectType() != typeid(float)) {
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : Error: Not available for the object type of the index. " 
+       << os->getObjectType().name();
+    operate_error_string_(ss, error);
+    return NULL;
+  }
   return static_cast<float*>(ngt_get_object(object_space, id, error));
 }
 
+NGTFloat16* ngt_get_object_as_float16(NGTObjectSpace object_space, ObjectID id, NGTError error) {
+  auto os = static_cast<NGT::ObjectSpace*>(object_space);
+  if (os->getObjectType() != typeid(NGT::float16)) {
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : Error: Not available for the object type of the index. " 
+       << os->getObjectType().name();
+    operate_error_string_(ss, error);
+    return NULL;
+  }
+  return static_cast<NGTFloat16*>(ngt_get_object(object_space, id, error));
+}
+
 uint8_t* ngt_get_object_as_integer(NGTObjectSpace object_space, ObjectID id, NGTError error) {
+  auto os = static_cast<NGT::ObjectSpace*>(object_space);
+  if (os->getObjectType() != typeid(uint8_t)) {
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : Error: Not available for the object type of the index. " 
+       << os->getObjectType().name();
+    operate_error_string_(ss, error);
+    return NULL;
+  }
   return static_cast<uint8_t*>(ngt_get_object(object_space, id, error));
 }
 
+float* ngt_get_allocated_object_as_float(NGTObjectSpace object_space, ObjectID id, NGTError error) {
+  auto objectSpace = static_cast<NGT::ObjectSpace*>(object_space);
+  std::vector<float> v;
+  try {
+    objectSpace->getObject(id, v);
+  } catch(std::exception &err) {
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what();
+    operate_error_string_(ss, error);
+    return NULL;
+  }
+  auto sizeOfObject = sizeof(float) * v.size();
+  auto fv = static_cast<float*>(malloc(sizeOfObject));  
+  if (fv == NULL) {
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : Error: Cannot allocate a vector.";
+    operate_error_string_(ss, error);
+    return NULL;
+  }
+  memcpy(fv, v.data(), sizeOfObject);
+  return fv;
+}
+
 void ngt_destroy_results(NGTObjectDistances results) {
     if(results == NULL) return;
     delete static_cast<NGT::ObjectDistances*>(results);

lib/NGT/Capi.h

@@ -31,6 +31,7 @@ typedef void* NGTObjectSpace;
 typedef void* NGTObjectDistances;
 typedef void* NGTError;
 typedef void* NGTOptimizer;
+typedef void NGTFloat16;
 
 typedef struct {
   ObjectID id;
@@ -145,6 +146,14 @@ ObjectID ngt_insert_index_as_float(NGTIndex, float*, uint32_t, NGTError);
 
 ObjectID ngt_append_index_as_float(NGTIndex, float*, uint32_t, NGTError);
 
+ObjectID ngt_insert_index_as_uint8(NGTIndex, uint8_t*, uint32_t, NGTError);
+
+ObjectID ngt_append_index_as_uint8(NGTIndex, uint8_t*, uint32_t, NGTError);
+
+ObjectID ngt_insert_index_as_float16(NGTIndex, NGTFloat16*, uint32_t, NGTError);
+
+ObjectID ngt_append_index_as_float16(NGTIndex, NGTFloat16*, uint32_t, NGTError);
+
 bool ngt_batch_append_index(NGTIndex, float*, uint32_t, NGTError);
 
 bool ngt_batch_insert_index(NGTIndex, float*, uint32_t, uint32_t *, NGTError);
@@ -159,8 +168,13 @@ void* ngt_get_object(NGTObjectSpace, ObjectID, NGTError);
 
 float* ngt_get_object_as_float(NGTObjectSpace, ObjectID, NGTError);
 
+NGTFloat16* ngt_get_object_as_float16(NGTObjectSpace, ObjectID, NGTError);
+
 uint8_t* ngt_get_object_as_integer(NGTObjectSpace, ObjectID, NGTError);
 
+// return an object as a float vector whatever the data type of the object.
+float* ngt_get_allocated_object_as_float(NGTObjectSpace, ObjectID, NGTError);
+
 void ngt_destroy_results(NGTObjectDistances);
 
 void ngt_destroy_property(NGTProperty);

lib/NGT/Clustering.h

@@ -892,7 +892,7 @@ namespace NGT {
       size_t count = 0;
       for (auto cit = clusters.begin(); cit != clusters.end(); ++cit) {    
 	count += (*cit).members.size();
-	double localD= 0.0;
+	double localD = 0.0;
 	for (auto mit = (*cit).members.begin(); mit != (*cit).members.end(); ++mit) {
 	  double distance = distanceL2((*cit).centroid, vectors[(*mit).vectorID]);
 	  d += distance;

lib/NGT/Graph.h

@@ -199,7 +199,7 @@ namespace NGT {
 	usedSize++;
       }
       size_t size() { return usedSize; }
-      size_t resize(size_t s) {
+      void resize(size_t s) {
 	if (s <= usedSize) {
 	  for (size_t i = s; i < usedSize; i++) {
 	    (*this)[i].first = 0;

lib/NGT/ObjectRepository.h

@@ -95,6 +95,13 @@ namespace NGT {
       abort();
     }
 
+#ifdef NGT_HALF_FLOAT
+    virtual PersistentObject *allocateNormalizedPersistentObject(const std::vector<float16> &obj) {
+      std::cerr << "ObjectRepository::allocateNormalizedPersistentObject(float): Fatal error! Something wrong!" << std::endl;
+      abort();
+    }
+#endif
+
     virtual PersistentObject *allocateNormalizedPersistentObject(const std::vector<uint8_t> &obj) {
       std::cerr << "ObjectRepository::allocateNormalizedPersistentObject(uint8_t): Fatal error! Something wrong!" << std::endl;
       abort();

lib/NGT/ObjectSpaceRepository.h

@@ -589,6 +589,15 @@ namespace NGT {
       }
       return allocatedObject;
     }
+#ifdef NGT_HALF_FLOAT
+    Object *allocateNormalizedObject(const std::vector<float16> &obj) {
+      Object *allocatedObject = ObjectRepository::allocateObject(obj);
+      if (normalization) {
+	normalize(*allocatedObject);
+      }
+      return allocatedObject;
+    }
+#endif
     Object *allocateNormalizedObject(const std::vector<uint8_t> &obj) {
       Object *allocatedObject = ObjectRepository::allocateObject(obj);
       if (normalization) {
@@ -611,15 +620,22 @@ namespace NGT {
       }
       return allocatedObject;
     }
-
     PersistentObject *allocateNormalizedPersistentObject(const std::vector<float> &obj) {
       PersistentObject *allocatedObject = ObjectRepository::allocatePersistentObject(obj);
       if (normalization) {
 	normalize(*allocatedObject);
       }
       return allocatedObject;
     }
-
+#ifdef NGT_HALF_FLOAT
+    PersistentObject *allocateNormalizedPersistentObject(const std::vector<float16> &obj) {
+      PersistentObject *allocatedObject = ObjectRepository::allocatePersistentObject(obj);
+      if (normalization) {
+	normalize(*allocatedObject);
+      }
+      return allocatedObject;
+    }
+#endif
     PersistentObject *allocateNormalizedPersistentObject(const std::vector<uint8_t> &obj) {
       PersistentObject *allocatedObject = ObjectRepository::allocatePersistentObject(obj);
       if (normalization) {

lib/NGT/defines.h.in

@@ -49,6 +49,10 @@
  #define	NGT_GRAPH_CHECK_VECTOR
 #endif
 
+#ifdef	NGT_SHARED_MEMORY_ALLOCATOR
+ #define	NGT_QBG_DISABLED
+#endif
+
 #if defined(NGT_AVX_DISABLED) 
 #define NGT_NO_AVX
 #elif defined(NGT_AVX2)