more C++ code fixing and polish for release (#275)

* more C++ code fixing and polish for release

* fixing for android build

* build flags for android release

* add missing exporting function

CMakeLists.txt

@@ -72,6 +72,8 @@ endif()
 # Build the libraries with -fPIC
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
+set_property(GLOBAL PROPERTY USE_FOLDERS ON)
+
 set(CMAKE_FIND_FRAMEWORK NEVER CACHE STRING "...")
 if(NOT "${CMAKE_FIND_FRAMEWORK}" STREQUAL "NEVER")
   message(FATAL_ERROR "CMAKE_FIND_FRAMEWORK is not NEVER")
@@ -114,8 +116,7 @@ if (OCOS_ENABLE_RE2_REGEX)
 endif()
 
 macro(standardize_output_folder bin_target)
-set_target_properties(${bin_target}
-  PROPERTIES
+set_target_properties(${bin_target} PROPERTIES
   RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin"
   LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib"
   ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib"
@@ -350,6 +351,7 @@ else()
   if (CMAKE_SYSTEM_NAME STREQUAL "Android")
     if (OCOS_ENABLE_SPM_TOKENIZER)
       target_link_libraries(ortcustomops PUBLIC log)
+      set_property(TARGET ortcustomops APPEND_STRING PROPERTY LINK_FLAGS "-Wl,-s -Wl,--version-script -Wl,${PROJECT_SOURCE_DIR}/shared/ortcustomops.ver")
     endif()
   endif()
   standardize_output_folder(ortcustomops)
@@ -382,6 +384,17 @@ if(OCOS_ENABLE_PYTHON)
   endif()
 endif()
 
+# clean up the requirements.txt files from 3rd party project folder to suppress the code security false alarms
+file(GLOB_RECURSE NO_USE_FILES ${CMAKE_BINARY_DIR}/_deps/*requirements.txt)
+message("Found the follow requirements.txt: ${NO_USE_FILES}")
+foreach(nf ${NO_USE_FILES})
+    file(TO_NATIVE_PATH ${nf} nf_native)
+    if (CMAKE_SYSTEM_NAME MATCHES "Windows")
+        execute_process(COMMAND cmd /c "del ${nf_native}")
+    else()
+        execute_process(COMMAND bash -c "rm ${nf_native}")
+    endif()
+endforeach()
 
 # test section
 if (OCOS_ENABLE_CTEST)

build.bat

@@ -11,10 +11,11 @@ IF "%VisualStudioVersion:~0,2%" == "16" GOTO :START_BUILD
 set GENERATOR="Visual Studio 17 2022"
 
 :START_BUILD
+set cmake_exe="%VSINSTALLDIR%Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe"
 mkdir .\out\Windows\ 2>NUL
-"%VSINSTALLDIR%Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe" -G %GENERATOR% -A x64 %* -B out\Windows -S .
+%cmake_exe% -G %GENERATOR% -A x64 %* -B out\Windows -S .
 IF %ERRORLEVEL% NEQ 0 EXIT /B %ERRORLEVEL%
-cmake --build out\Windows --config RelWithDebInfo
+%cmake_exe% --build out\Windows --config RelWithDebInfo
 IF %ERRORLEVEL% NEQ 0 EXIT /B %ERRORLEVEL%
 GOTO :EOF
 

cmake/externals/blingfire.cmake

@@ -12,12 +12,8 @@ if (NOT blingfire_POPULATED)
 
     # enable size optimization build
     add_subdirectory(${blingfire_SOURCE_DIR} ${blingfire_BINARY_DIR} EXCLUDE_FROM_ALL)
-    # we don't use any Python code from Blingfire codebase
-    set(NO_USE_FILE ${blingfire_SOURCE_DIR}/scripts/requirements.txt)
-    file(TO_NATIVE_PATH ${NO_USE_FILE} NO_USE_FILE)
-    if (CMAKE_SYSTEM_NAME MATCHES "Windows")
-        execute_process(COMMAND cmd /c "del ${NO_USE_FILE}")
-    else()
-        execute_process(COMMAND bash -c "rm ${NO_USE_FILE}")
-    endif()
+    set_target_properties(bingfirtinydll_static PROPERTIES
+        FOLDER externals/bingfire)
+    set_target_properties(fsaClientTiny PROPERTIES
+        FOLDER externals/bingfire)
 endif()

cmake/externals/googlere2.cmake

@@ -11,5 +11,6 @@ if(NOT ${lcName}_POPULATED)
   add_subdirectory(${googlere2_SOURCE_DIR} ${googlere2_BINARY_DIR} EXCLUDE_FROM_ALL)
   set_target_properties(re2
     PROPERTIES
-        POSITION_INDEPENDENT_CODE ON)
+        POSITION_INDEPENDENT_CODE ON
+        FOLDER externals/google/re2)
 endif()

cmake/externals/googletest.cmake

@@ -5,3 +5,7 @@ FetchContent_Declare(
 )
 
 FetchContent_MakeAvailable(googletest)
+set_target_properties(gmock PROPERTIES FOLDER "externals/gtest")
+set_target_properties(gmock_main PROPERTIES FOLDER "externals/gtest")
+set_target_properties(gtest PROPERTIES FOLDER "externals/gtest")
+set_target_properties(gtest_main PROPERTIES FOLDER "externals/gtest")

cmake/externals/opencv.cmake

@@ -121,3 +121,12 @@ endif()
 
 # unset it to avoid affecting other projects.
 unset(EXECUTABLE_OUTPUT_PATH CACHE)
+
+if (CMAKE_SYSTEM_NAME MATCHES "Windows")
+    set(opencv_projs gen_opencv_java_source gen_opencv_js_source gen_opencv_python_source)
+    list(APPEND opencv_projs gen_opencv_objc_source gen_opencv_objc_source_ios gen_opencv_objc_source_osx)
+    list(APPEND opencv_projs opencv_highgui_plugins opencv_videoio_plugins)
+    foreach(p ${opencv_projs})
+        set_target_properties(${p} PROPERTIES FOLDER "externals/opencv")
+    endforeach()
+endif()

cmake/externals/sentencepieceproject.cmake

@@ -9,6 +9,8 @@ FetchContent_GetProperties(spm)
 if(NOT spm_POPULATED)
   FetchContent_Populate(spm)
   add_subdirectory(${spm_SOURCE_DIR} ${spm_BINARY_DIR} EXCLUDE_FROM_ALL)
+  set_target_properties(sentencepiece-static PROPERTIES
+    FOLDER externals/google/sentencepiece)
 endif()
 
 set(spm_INCLUDE_DIRS

cmake/noexcep_ops.cmake

@@ -15,3 +15,6 @@ endif()
 if (OCOS_ENABLE_SPM_TOKENIZER)
     message(FATAL_ERROR "SPM_TOKENIZER operator needs c++ exceptions support")
 endif()
+if (OCOS_ENABLE_OPENCV)
+    message(FATAL_ERROR "the operators depending on opencv needs c++ exceptions support")
+endif()

onnxruntime_extensions/_ortapi2.py

@@ -17,7 +17,8 @@ def get_opset_version_from_ort():
         "1.8": 14,
         "1.9": 15,
         "1.10": 15,
-        "1.11": 16
+        "1.11": 16,
+        "1.12": 17
     }
 
     ort_ver_string = '.'.join(_ort.__version__.split('.')[0:2])

onnxruntime_extensions/pnp/_onnx_ops.py

@@ -13,7 +13,7 @@
 OPSET_TO_IR_VERSION = {
     1: 3, 2: 3, 3: 3, 4: 3, 5: 3, 6: 3,
     7: 3, 8: 3, 9: 4, 10: 5, 11: 6, 12: 7,
-    13: 7, 14: 7, 15: 8, 16: 8
+    13: 7, 14: 7, 15: 8, 16: 8, 17: 8
 }
 
 

operators/text/op_equal_impl.hpp

@@ -39,12 +39,12 @@ class BroadcastIteratorRight {
   struct BroadcastIteratorRightState {
     const BroadcastIteratorRight<T1, T2, T3>* parent = nullptr;
     std::vector<int64_t> index1;
-    const T1* p1;
-    const T1* end_;
-    const T2* p2;
-    T3* p3;
-    size_t last;
-    int dim;
+    const T1* p1 = nullptr;
+    const T1* end_ = nullptr;
+    const T2* p2 = nullptr;
+    T3* p3 = nullptr;
+    size_t last = 0;
+    int dim = -1;
 
     void init(const BroadcastIteratorRight<T1, T2, T3>& p) {
       parent = &p;

shared/ortcustomops.ver

@@ -0,0 +1,4 @@
+{ global:
+RegisterCustomOps;
+AddExternalCustomOp;
+local: *; };

test/shared_test/test_kernel.hpp

@@ -9,7 +9,7 @@ const char* GetLibraryPath();
 
 struct TestValue {
   const char* name = nullptr;
-  ONNXTensorElementDataType element_type;
+  ONNXTensorElementDataType element_type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED;
   std::vector<int64_t> dims;
   std::vector<float> values_float;
   std::vector<int32_t> values_int32;