Add patch to fix segfault in nmslib during ingestion

Signed-off-by: John Mazanec <[email protected]>
opensearch-project · Mar 14, 2024 · 2d9abf2 · 2d9abf2
1 parent bfcf7dc
commit 2d9abf2
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 0 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -47,6 +47,9 @@ jobs:
           rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch
           git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch
           rm ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch
+          cd ../nmslib
+          git apply --ignore-space-change --ignore-whitespace --3way ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch
+          rm ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch
         working-directory: ${{ github.workspace }}
 
       - name: Setup Java ${{ matrix.java }}

diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml
@@ -47,6 +47,9 @@ jobs:
           rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch
           git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch
           rm ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch
+          cd ../nmslib
+          git apply --ignore-space-change --ignore-whitespace --3way ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch
+          rm ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch
         working-directory: ${{ github.workspace }}
 
       - name: Setup Java ${{ matrix.java }}

diff --git a/jni/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch b/jni/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch
@@ -0,0 +1,31 @@
+From aa1ca485c0ab8b79dae1fb5c1567149c5f61b533 Mon Sep 17 00:00:00 2001
+From: John Mazanec <[email protected]>
+Date: Thu, 14 Mar 2024 12:22:06 -0700
+Subject: [PATCH] Initialize maxlevel during add from enterpoint->level
+
+Signed-off-by: John Mazanec <[email protected]>
+---
+ similarity_search/src/method/hnsw.cc | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/similarity_search/src/method/hnsw.cc b/similarity_search/src/method/hnsw.cc
+index 35b372c..e9a725e 100644
+--- a/similarity_search/src/method/hnsw.cc
++++ b/similarity_search/src/method/hnsw.cc
+@@ -542,8 +542,12 @@ namespace similarity {
+
+         NewElement->init(curlevel, maxM_, maxM0_);
+
+-        int maxlevelcopy = maxlevel_;
++        // Get the enterpoint at this moment and then use it to set the 
++        // max level that is used. Copying maxlevel from this->maxlevel_
++        // can lead to race conditions during concurrent insertion. See:
++        // https://github.com/nmslib/nmslib/issues/544
+         HnswNode *ep = enterpoint_;
++        int maxlevelcopy = ep->level; 
+         if (curlevel < maxlevelcopy) {
+             const Object *currObj = ep->getData();
+
+-- 
+2.39.3 (Apple Git-146)
+