Merge branch 'main' into hit_queue_size

Signed-off-by: John Mazanec <[email protected]>

.github/workflows/CI.yml

@@ -45,8 +45,8 @@ jobs:
           cd jni/external/faiss
           git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch
           rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch
-          git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch
-          rm ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch
+          git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch
+          rm ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch
           cd ../nmslib
           git apply --ignore-space-change --ignore-whitespace --3way ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch
           rm ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch

.github/workflows/test_security.yml

@@ -45,8 +45,8 @@ jobs:
           cd jni/external/faiss
           git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch
           rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch
-          git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch
-          rm ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch
+          git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch
+          rm ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch
           cd ../nmslib
           git apply --ignore-space-change --ignore-whitespace --3way ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch
           rm ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch

CHANGELOG.md

@@ -18,12 +18,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 * Optize Faiss Query With Filters: Reduce iteration and memory for id filter [#1402](https://github.com/opensearch-project/k-NN/pull/1402)
 * Detect AVX2 Dynamically on the System [#1502](https://github.com/opensearch-project/k-NN/pull/1502)
 * Validate zero vector when using cosine metric [#1501](https://github.com/opensearch-project/k-NN/pull/1501)
-* Persist model definition in model metadata [#1527](https://github.com/opensearch-project/k-NN/pull/1527)
+* Persist model definition in model metadata [#1527] (https://github.com/opensearch-project/k-NN/pull/1527)
+* Added Inner Product Space type support for Lucene Engine [#1551](https://github.com/opensearch-project/k-NN/pull/1551)
 * Make the HitQueue size more appropriate for exact search [#1549](https://github.com/opensearch-project/k-NN/pull/1549)
 ### Bug Fixes
 * Disable sdc table for HNSWPQ read-only indices [#1518](https://github.com/opensearch-project/k-NN/pull/1518)
 * Switch SpaceType.INNERPRODUCT's vector similarity function to MAXIMUM_INNER_PRODUCT [#1532](https://github.com/opensearch-project/k-NN/pull/1532)
 * Add patch to fix arm segfault in nmslib during ingestion [#1541](https://github.com/opensearch-project/k-NN/pull/1541)
+* Share ivfpq-l2 table allocations across indices on load [#1558](https://github.com/opensearch-project/k-NN/pull/1558)
 ### Infrastructure
 * Manually install zlib for win CI [#1513](https://github.com/opensearch-project/k-NN/pull/1513)
 * Update k-NN build artifact script to enable SIMD on ARM for Faiss [#1543](https://github.com/opensearch-project/k-NN/pull/1543)

jni/CMakeLists.txt

@@ -168,13 +168,13 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S
     endif ()
 
     # Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu.
-    find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH)
+    find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0002-Enable-precomp-table-to-be-shared-ivfpq.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH)
 
     # If it exists, apply patches
     if (EXISTS ${PATCH_FILE})
         message(STATUS "Applying custom patches.")
         execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE)
-
+        execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE)
         if(RESULT_CODE)
             message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}")
         endif()

jni/include/faiss_wrapper.h

@@ -33,6 +33,18 @@ namespace knn_jni {
         // Return a pointer to the loaded index
         jlong LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jstring indexPathJ);
 
+        // Check if a loaded index requires shared state
+        bool IsSharedIndexStateRequired(jlong indexPointerJ);
+
+        // Initializes the shared index state from an index. Note, this will not set the state for
+        // the index pointed to by indexPointerJ. To set it, SetSharedIndexState needs to be called.
+        //
+        // Return a pointer to the shared index state
+        jlong InitSharedIndexState(jlong indexPointerJ);
+
+        // Sets the sharedIndexState for an index
+        void SetSharedIndexState(jlong indexPointerJ, jlong shareIndexStatePointerJ);
+
         // Execute a query against the index located in memory at indexPointerJ.
         //
         // Return an array of KNNQueryResults
@@ -49,6 +61,9 @@ namespace knn_jni {
         // Free the index located in memory at indexPointerJ
         void Free(jlong indexPointer);
 
+        // Free shared index state in memory at shareIndexStatePointerJ
+        void FreeSharedIndexState(jlong shareIndexStatePointerJ);
+
         // Perform initilization operations for the library
         void InitLibrary();
 

jni/include/org_opensearch_knn_jni_FaissService.h

@@ -42,18 +42,42 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndexFromT
 JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndex
   (JNIEnv *, jclass, jstring);
 
+/*
+ * Class:     org_opensearch_knn_jni_FaissService
+ * Method:    isSharedIndexStateRequired
+ * Signature: (J)Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_opensearch_knn_jni_FaissService_isSharedIndexStateRequired
+  (JNIEnv *, jclass, jlong);
+
+/*
+ * Class:     org_opensearch_knn_jni_FaissService
+ * Method:    initSharedIndexState
+ * Signature: (J)J
+ */
+JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initSharedIndexState
+  (JNIEnv *, jclass, jlong);
+
+/*
+ * Class:     org_opensearch_knn_jni_FaissService
+ * Method:    setSharedIndexState
+ * Signature: (JJ)V
+ */
+JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_setSharedIndexState
+  (JNIEnv *, jclass, jlong, jlong);
+
 /*
  * Class:     org_opensearch_knn_jni_FaissService
  * Method:    queryIndex
- * Signature: (J[FI)[Lorg/opensearch/knn/index/query/KNNQueryResult;
+ * Signature: (J[FI[I)[Lorg/opensearch/knn/index/query/KNNQueryResult;
  */
 JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndex
   (JNIEnv *, jclass, jlong, jfloatArray, jint, jintArray);
 
 /*
  * Class:     org_opensearch_knn_jni_FaissService
- * Method:    queryIndex_WithFilter
- * Signature: (J[FI[J)[Lorg/opensearch/knn/index/query/KNNQueryResult;
+ * Method:    queryIndexWithFilter
+ * Signature: (J[FI[JI[I)[Lorg/opensearch/knn/index/query/KNNQueryResult;
  */
 JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndexWithFilter
   (JNIEnv *, jclass, jlong, jfloatArray, jint, jlongArray, jint, jintArray);
@@ -66,6 +90,14 @@ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryInd
 JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_free
   (JNIEnv *, jclass, jlong);
 
+/*
+ * Class:     org_opensearch_knn_jni_FaissService
+ * Method:    freeSharedIndexState
+ * Signature: (J)V
+ */
+JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_freeSharedIndexState
+  (JNIEnv *, jclass, jlong);
+
 /*
  * Class:     org_opensearch_knn_jni_FaissService
  * Method:    initLibrary