Fix python version (#333)

MaggieQi · cheqi · web-flow · commit dda1180a8e58 · 2022-12-02T14:44:11.000+08:00
* update python to python3

* use dynamic linking

* enable ANNIndexTestTool code compiling

* add python version in wheel package

* update nuspec

* enable to set different maxcheck and hashexponent

* change to configure python version

* trigger azurepipeline

* trigger

* fix python version

* fix python version

* fix python version in windows

* fix cosine kmeans

* clean avx/see header files

* fix nuspec

Co-authored-by: cheqi &lt;cheqi@SRGSSD-07&gt;
diff --git a/AnnService.users.props b/AnnService.users.props
@@ -10,9 +10,10 @@
 
   <PropertyGroup Label="UserMacros">
 	<OutAppDir>$(SolutionDir)\$(Platform)\$(Configuration)\</OutAppDir>
-    <OutLibDir>$(SolutionDir)\$(Platform)\$(Configuration)\</OutLibDir>
+	<OutLibDir>$(SolutionDir)\$(Platform)\$(Configuration)\</OutLibDir>
 	<JavaIncDir></JavaIncDir>
 	<JavaLib></JavaLib>
+  <PYTHON_VERSION>3.9</PYTHON_VERSION>
   </PropertyGroup>
 </Project>
 
diff --git a/AnnService/CoreLibrary.vcxproj b/AnnService/CoreLibrary.vcxproj
@@ -128,6 +128,11 @@
       <PreprocessorDefinitions>_MBCS;_SCL_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <AdditionalOptions>/Zc:twoPhase- /Zc:__cplusplus %(AdditionalOptions)</AdditionalOptions>
       <LanguageStandard>stdcpp17</LanguageStandard>
+      <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
+      <InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <FloatingPointModel>Fast</FloatingPointModel>
     </ClCompile>
     <Link>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
diff --git a/AnnService/CoreLibrary.vcxproj.filters b/AnnService/CoreLibrary.vcxproj.filters
@@ -282,15 +282,18 @@
     <ClCompile Include="src\Core\Common\CommonUtils.cpp">
       <Filter>Source Files\Core\Common</Filter>
     </ClCompile>
+    <ClCompile Include="src\Core\Common\SIMDUtils.cpp">
+      <Filter>Source Files\Core\Common</Filter>
+    </ClCompile>
+    <ClCompile Include="src\Core\Common\TruthSet.cpp">
+      <Filter>Source Files\Core\Common</Filter>
+    </ClCompile>
     <ClCompile Include="src\Core\SPANN\SPANNIndex.cpp">
       <Filter>Source Files\Core\SPANN</Filter>
     </ClCompile>
     <ClCompile Include="src\Helper\AsyncFileReader.cpp">
       <Filter>Source Files\Helper</Filter>
     </ClCompile>
-    <ClCompile Include="src\Core\Common\SIMDUtils.cpp">
-      <Filter>Source Files\Core\Common</Filter>
-    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />
diff --git a/AnnService/SSDServing.vcxproj b/AnnService/SSDServing.vcxproj
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+﻿<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|Win32">
@@ -161,6 +161,12 @@
       <OpenMPSupport>true</OpenMPSupport>
       <AdditionalOptions>/Zc:twoPhase- %(AdditionalOptions)</AdditionalOptions>
       <LanguageStandard>stdcpp17</LanguageStandard>
+      <ProgramDataBaseFileName>$(IntDir)$(ProjectName).pdb</ProgramDataBaseFileName>
+      <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
+      <InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <FloatingPointModel>Fast</FloatingPointModel>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
diff --git a/AnnService/inc/Core/Common.h b/AnnService/inc/Core/Common.h
@@ -81,6 +81,15 @@ inline bool fileexists(const TCHAR* path) {
     return (dwAttr != INVALID_FILE_ATTRIBUTES) && (dwAttr & FILE_ATTRIBUTE_DIRECTORY) == 0;
 }
 #define mkdir(a) CreateDirectory(a, NULL)
+
+#ifndef max
+#define max(a,b) (((a) > (b)) ? (a) : (b))
+#endif
+
+#ifndef min
+#define min(a,b) (((a) < (b)) ? (a) : (b))
+#endif
+
 #endif
 
 namespace SPTAG
diff --git a/AnnService/inc/Core/Common/BKTree.h b/AnnService/inc/Core/Common/BKTree.h
@@ -57,10 +57,6 @@ namespace SPTAG
                     _RD = m_pQuantizer->ReconstructDim();
                     fComputeDistance = m_pQuantizer->DistanceCalcSelector<T>(distMethod);
                 }
-                else if (distMethod == DistCalcMethod::L2 || distMethod == DistCalcMethod::Cosine)
-                {
-                    fComputeDistance = COMMON::DistanceCalcSelector<T>(DistCalcMethod::L2);
-                }
                 else {
                     fComputeDistance = COMMON::DistanceCalcSelector<T>(distMethod);
                 }
@@ -182,20 +178,20 @@ namespace SPTAG
                     for (DimensionType j = 0; j < args._RD; j++) {
                         currCenters[j] /= args.counts[k];
                     }
-                    /*
+                    
                     if (args._M == DistCalcMethod::Cosine) {
                         COMMON::Utils::Normalize(currCenters, args._RD, COMMON::Utils::GetBase<T>());
                     }
-                    */
+                    
                     if (args.m_pQuantizer) {
                         for (DimensionType j = 0; j < args._RD; j++) reconstructVector[j] = (R)(currCenters[j]);
                         args.m_pQuantizer->QuantizeVector(reconstructVector.data(), (uint8_t*)TCenter);
                     }
                     else {
-                        for (DimensionType j = 0; j < args._RD; j++) TCenter[j] = (T)(currCenters[j]);
+                        for (DimensionType j = 0; j < args._D; j++) TCenter[j] = (T)(currCenters[j]);
                     }
                 }
-                diff += args.fComputeDistance(args.centers + k*args._D, TCenter, args._D);
+                diff += DistanceUtils::ComputeDistance(TCenter, args.centers + k * args._D, args._D, DistCalcMethod::L2);
             }
             return diff;
         }
diff --git a/AnnService/inc/Core/Common/DistanceUtils.h b/AnnService/inc/Core/Common/DistanceUtils.h
@@ -4,7 +4,6 @@
 #ifndef _SPTAG_COMMON_DISTANCEUTILS_H_
 #define _SPTAG_COMMON_DISTANCEUTILS_H_
 
-#include <xmmintrin.h>
 #include <functional>
 #include <iostream>
 
diff --git a/AnnService/inc/Core/Common/InstructionUtils.h b/AnnService/inc/Core/Common/InstructionUtils.h
@@ -8,6 +8,9 @@
 
 #ifndef _MSC_VER
 #include <cpuid.h>
+#include <xmmintrin.h>
+#include <immintrin.h>
+
 void cpuid(int info[4], int InfoType);
 
 #else
diff --git a/AnnService/inc/Core/Common/PQQuantizer.h b/AnnService/inc/Core/Common/PQQuantizer.h
@@ -173,7 +173,7 @@ namespace SPTAG
                         subcodebooks += m_DimPerSubvector;
                     }
                     assert(bestIndex != -1);
-                    vecout[i] = bestIndex;
+                    vecout[i] = (std::uint8_t)bestIndex;
                     subvec += m_DimPerSubvector;
                 }
             }           
diff --git a/AnnService/inc/Core/Common/RelativeNeighborhoodGraph.h b/AnnService/inc/Core/Common/RelativeNeighborhoodGraph.h
@@ -4,7 +4,6 @@
 #ifndef _SPTAG_COMMON_RNG_H_
 #define _SPTAG_COMMON_RNG_H_
 
-#include <xmmintrin.h>
 #include "NeighborhoodGraph.h"
 
 namespace SPTAG
diff --git a/AnnService/inc/Core/Common/SIMDUtils.h b/AnnService/inc/Core/Common/SIMDUtils.h
@@ -4,7 +4,6 @@
 #ifndef _SPTAG_COMMON_SIMDUTILS_H_
 #define _SPTAG_COMMON_SIMDUTILS_H_
 
-#include <xmmintrin.h>
 #include <functional>
 #include <iostream>
 
diff --git a/AnnService/inc/Core/SPANN/ExtraFullGraphSearcher.h b/AnnService/inc/Core/SPANN/ExtraFullGraphSearcher.h
@@ -148,7 +148,7 @@ namespace SPTAG
 #endif
 */
 #else
-                        (p_opt.m_searchPostingPageLimit + 1) * PageSize, 2, 2, p_opt.m_ioThreads
+                        (p_opt.m_searchPostingPageLimit + 1) * PageSize, 2, 2, (std::uint16_t)p_opt.m_ioThreads
 #endif
                     )) {
                         LOG(Helper::LogLevel::LL_Error, "Cannot open file:%s!\n", curFile.c_str());
@@ -735,7 +735,7 @@ namespace SPTAG
                 }
 
                 auto t5 = std::chrono::high_resolution_clock::now();
-                double elapsedSeconds = std::chrono::duration_cast<std::chrono::seconds>(t5 - t1).count();
+                auto elapsedSeconds = std::chrono::duration_cast<std::chrono::seconds>(t5 - t1).count();
                 LOG(Helper::LogLevel::LL_Info, "Total used time: %.2lf minutes (about %.2lf hours).\n", elapsedSeconds / 60.0, elapsedSeconds / 3600.0);
              
                 return true;
@@ -917,7 +917,7 @@ namespace SPTAG
 
             inline void ParseDeltaEncoding(std::shared_ptr<VectorIndex>& p_index, ListInfo* p_info, ValueType* vector)
             {
-                ValueType* headVector = (ValueType*)p_index->GetSample(p_info - m_listInfos.data());
+                ValueType* headVector = (ValueType*)p_index->GetSample((SizeType)(p_info - m_listInfos.data()));
                 COMMON::SIMDUtils::ComputeSum(vector, headVector, m_iDataDimension);
             }
 
diff --git a/AnnService/inc/Core/SPANN/IExtraSearcher.h b/AnnService/inc/Core/SPANN/IExtraSearcher.h
@@ -121,6 +121,9 @@ namespace SPTAG {
                     m_pageBuffers[pi].ReservePageBuffer(p_maxPages);
                 }
                 m_diskRequests.resize(p_internalResultNum);
+                for (int pi = 0; pi < p_internalResultNum; pi++) {
+                    m_diskRequests[pi].m_extension = m_processIocp.handle();
+                }
                 m_enableDataCompression = enableDataCompression;
                 if (enableDataCompression) {
                     m_decompressBuffer.ReservePageBuffer(p_maxPages);
diff --git a/AnnService/inc/Helper/AsyncFileReader.h b/AnnService/inc/Helper/AsyncFileReader.h
@@ -128,6 +128,10 @@ namespace SPTAG
                 return true;
             }
 
+            void* handle() {
+                return m_handle.GetHandle();
+            }
+
         private:
             HandleWrapper m_handle;
         };
@@ -160,17 +164,13 @@ namespace SPTAG
 
                 PreAllocQueryContext();
 
-#ifndef BATCH_READ
                 int iocpThreads = threadPoolSize;
                 m_fileIocp.Reset(::CreateIoCompletionPort(m_fileHandle.GetHandle(), NULL, NULL, iocpThreads));
                 for (int i = 0; i < iocpThreads; ++i)
                 {
                     m_fileIocpThreads.emplace_back(std::thread(std::bind(&AsyncFileIO::ListionIOCP, this)));
                 }
                 return m_fileIocp.IsValid();
-#else
-                return true;
-#endif
             }
 
             virtual std::uint64_t ReadBinary(std::uint64_t readSize, char* buffer, std::uint64_t offset = UINT64_MAX)
@@ -239,46 +239,27 @@ namespace SPTAG
 
             virtual bool BatchReadFile(AsyncReadRequest* readRequests, std::uint32_t requestCount)
             {
-                std::vector<ResourceType*> waitResources;
+                std::uint32_t remaining = requestCount;
+                HANDLE completeQueue = readRequests[0].m_extension;
                 for (std::uint32_t i = 0; i < requestCount; i++) {
                     AsyncReadRequest* readRequest = &(readRequests[i]);
-
-                    ResourceType* resource = GetResource();
-                    DiskUtils::CallbackOverLapped& col = resource->m_col;
-                    memset(&col, 0, sizeof(col));
-                    col.Offset = (readRequest->m_offset & 0xffffffff);
-                    col.OffsetHigh = (readRequest->m_offset >> 32);
-                    col.m_data = (void*)readRequest;
-
-                    col.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
-                    if (!::ReadFile(m_fileHandle.GetHandle(),
-                        readRequest->m_buffer,
-                        static_cast<DWORD>(readRequest->m_readSize),
-                        nullptr,
-                        &col) && GetLastError() != ERROR_IO_PENDING)
-                    {
-                        if (col.hEvent) CloseHandle(col.hEvent);
-                        ReturnResource(resource);
-                        LOG(Helper::LogLevel::LL_Error, "Cannot read request %d error:%u\n", i, GetLastError());
-                    }
-                    else {
-                        waitResources.push_back(resource);
-                    }
+                    if (!ReadFileAsync(*readRequest)) remaining--;
                 }
 
-                for (int i = 0; i < waitResources.size(); i++) {
-                    DWORD readbytes = 0;
-                    DiskUtils::CallbackOverLapped* col = &(waitResources[i]->m_col);
-                    AsyncReadRequest* readRequest = (AsyncReadRequest*)(col->m_data);
+                while (remaining > 0) {
+                    DWORD cBytes;
+                    ULONG_PTR key;
+                    OVERLAPPED* ol;
+                    BOOL ret = ::GetQueuedCompletionStatus(completeQueue,
+                        &cBytes,
+                        &key,
+                        &ol,
+                        INFINITE);
+                    if (FALSE == ret || nullptr == ol) break;
+                    remaining--;
+                    AsyncReadRequest* req = reinterpret_cast<AsyncReadRequest*>(ol);
+                    req->m_callback(true);
 
-                    if (!::GetOverlappedResult(m_fileHandle.GetHandle(), col, &readbytes, TRUE) || readbytes != readRequest->m_readSize) {
-                        LOG(Helper::LogLevel::LL_Error, "Cannot wait request %d readbytes:%llu expect:%llu error:%u\n", i, readbytes, readRequest->m_readSize, GetLastError());
-                    }
-                    else {
-                        readRequest->m_success = true;
-                    }
-                    if (col->hEvent) CloseHandle(col->hEvent);
-                    ReturnResource(waitResources[i]);
                 }
                 return true;
             }
@@ -420,7 +401,7 @@ namespace SPTAG
 
                     if (nullptr != req)
                     {
-                        req->m_callback(true);
+                        ::PostQueuedCompletionStatus(req->m_extension, 0, NULL, reinterpret_cast<LPOVERLAPPED>(req));
                     }
                 }
             }
@@ -512,6 +493,11 @@ namespace SPTAG
                 return true;
             }
 
+            void* handle() 
+            {
+                return nullptr;
+            }
+
         protected:
             int m_front, m_end, m_capacity;
             std::unique_ptr<AsyncReadRequest* []> m_queue;
diff --git a/AnnService/src/Core/Common/DistanceUtils.cpp b/AnnService/src/Core/Common/DistanceUtils.cpp
@@ -2,7 +2,6 @@
 // Licensed under the MIT License.
 
 #include "inc/Core/Common/DistanceUtils.h"
-#include <immintrin.h>
 
 using namespace SPTAG;
 using namespace SPTAG::COMMON;
diff --git a/AnnService/src/Core/Common/SIMDUtils.cpp b/AnnService/src/Core/Common/SIMDUtils.cpp
@@ -2,7 +2,6 @@
 // Licensed under the MIT License.
 
 #include "inc/Core/Common/SIMDUtils.h"
-#include <immintrin.h>
 
 using namespace SPTAG;
 using namespace SPTAG::COMMON;
diff --git a/AnnService/src/Core/SPANN/SPANNIndex.cpp b/AnnService/src/Core/SPANN/SPANNIndex.cpp
@@ -82,8 +82,8 @@ namespace SPTAG
             if (m_index->LoadIndexDataFromMemory(p_indexBlobs) != ErrorCode::Success) return ErrorCode::Fail;
 
             m_index->SetParameter("NumberOfThreads", std::to_string(m_options.m_iSSDNumberOfThreads));
-            m_index->SetParameter("MaxCheck", std::to_string(m_options.m_maxCheck));
-            m_index->SetParameter("HashTableExponent", std::to_string(m_options.m_hashExp));
+            //m_index->SetParameter("MaxCheck", std::to_string(m_options.m_maxCheck));
+            //m_index->SetParameter("HashTableExponent", std::to_string(m_options.m_hashExp));
             m_index->UpdateIndex();
             m_index->SetReady(true);
 
@@ -113,8 +113,8 @@ namespace SPTAG
             if (m_index->LoadIndexData(p_indexStreams) != ErrorCode::Success) return ErrorCode::Fail;
 
             m_index->SetParameter("NumberOfThreads", std::to_string(m_options.m_iSSDNumberOfThreads));
-            m_index->SetParameter("MaxCheck", std::to_string(m_options.m_maxCheck));
-            m_index->SetParameter("HashTableExponent", std::to_string(m_options.m_hashExp));
+            //m_index->SetParameter("MaxCheck", std::to_string(m_options.m_maxCheck));
+            //m_index->SetParameter("HashTableExponent", std::to_string(m_options.m_hashExp));
             m_index->UpdateIndex();
             m_index->SetReady(true);
 
@@ -842,8 +842,8 @@ namespace SPTAG
         {
             omp_set_num_threads(m_options.m_iSSDNumberOfThreads);
             m_index->SetParameter("NumberOfThreads", std::to_string(m_options.m_iSSDNumberOfThreads));
-            m_index->SetParameter("MaxCheck", std::to_string(m_options.m_maxCheck));
-            m_index->SetParameter("HashTableExponent", std::to_string(m_options.m_hashExp));
+            //m_index->SetParameter("MaxCheck", std::to_string(m_options.m_maxCheck));
+            //m_index->SetParameter("HashTableExponent", std::to_string(m_options.m_hashExp));
             m_index->UpdateIndex();
             m_workSpacePool.reset(new COMMON::WorkSpacePool<ExtraWorkSpace>());
             m_workSpacePool->Init(m_options.m_iSSDNumberOfThreads, m_options.m_maxCheck, m_options.m_hashExp, m_options.m_searchInternalResultNum, max(m_options.m_postingPageLimit, m_options.m_searchPostingPageLimit + 1) << PageSizeEx, int(m_options.m_enableDataCompression));
diff --git a/AnnService/src/Helper/AsyncFileReader.cpp b/AnnService/src/Helper/AsyncFileReader.cpp
diff --git a/MANIFEST.in b/MANIFEST.in
diff --git a/SPTAG.nuspec b/SPTAG.nuspec
diff --git a/Wrappers/CMakeLists.txt b/Wrappers/CMakeLists.txt
diff --git a/Wrappers/PythonClient.vcxproj b/Wrappers/PythonClient.vcxproj
diff --git a/Wrappers/PythonCore.vcxproj b/Wrappers/PythonCore.vcxproj
diff --git a/Wrappers/packages.config b/Wrappers/packages.config
diff --git a/setup.py b/setup.py

Original file line number	Diff line number	Diff line change
`@@ -57,10 +57,6 @@ namespace SPTAG`
`57`	`57`	`_RD = m_pQuantizer->ReconstructDim();`
`58`	`58`	`fComputeDistance = m_pQuantizer->DistanceCalcSelector<T>(distMethod);`
`59`	`59`	`}`
`60`		`- else if (distMethod == DistCalcMethod::L2 \|\| distMethod == DistCalcMethod::Cosine)`
`61`		`- {`
`62`		`- fComputeDistance = COMMON::DistanceCalcSelector<T>(DistCalcMethod::L2);`
`63`		`- }`
`64`	`60`	`else {`
`65`	`61`	`fComputeDistance = COMMON::DistanceCalcSelector<T>(distMethod);`
`66`	`62`	`}`
`@@ -182,20 +178,20 @@ namespace SPTAG`
`182`	`178`	`for (DimensionType j = 0; j < args._RD; j++) {`
`183`	`179`	`currCenters[j] /= args.counts[k];`
`184`	`180`	`}`
`185`		`- /*`
	`181`	`+`
`186`	`182`	`if (args._M == DistCalcMethod::Cosine) {`
`187`	`183`	`COMMON::Utils::Normalize(currCenters, args._RD, COMMON::Utils::GetBase<T>());`
`188`	`184`	`}`
`189`		`- */`
	`185`	`+`
`190`	`186`	`if (args.m_pQuantizer) {`
`191`	`187`	`for (DimensionType j = 0; j < args._RD; j++) reconstructVector[j] = (R)(currCenters[j]);`
`192`	`188`	`args.m_pQuantizer->QuantizeVector(reconstructVector.data(), (uint8_t*)TCenter);`
`193`	`189`	`}`
`194`	`190`	`else {`
`195`		`- for (DimensionType j = 0; j < args._RD; j++) TCenter[j] = (T)(currCenters[j]);`
	`191`	`+ for (DimensionType j = 0; j < args._D; j++) TCenter[j] = (T)(currCenters[j]);`
`196`	`192`	`}`
`197`	`193`	`}`
`198`		`- diff += args.fComputeDistance(args.centers + k*args._D, TCenter, args._D);`
	`194`	`+ diff += DistanceUtils::ComputeDistance(TCenter, args.centers + k * args._D, args._D, DistCalcMethod::L2);`
`199`	`195`	`}`
`200`	`196`	`return diff;`
`201`	`197`	`}`
Original file line number	Diff line number	Diff line change
`@@ -173,7 +173,7 @@ namespace SPTAG`
`173`	`173`	`subcodebooks += m_DimPerSubvector;`
`174`	`174`	`}`
`175`	`175`	`assert(bestIndex != -1);`
`176`		`- vecout[i] = bestIndex;`
	`176`	`+ vecout[i] = (std::uint8_t)bestIndex;`
`177`	`177`	`subvec += m_DimPerSubvector;`
`178`	`178`	`}`
`179`	`179`	`}`