Skip to content

Commit 620050f

Browse files
wenbinglwenbingl
authored
reimplement resize cpu kernel for image processing (microsoft#768)
* reimplement resize cpu kernel for image processing * accuracy fixing and code refinement * fix the build issues * fix Linux build issue * more fixings * Fix the pipeline issue * fix the ci script * try to fix CUDA machine pool
1 parent d79299e commit 620050f

22 files changed

+1818
-126
lines changed

‎.pipelines/ci.yml

-4
Original file line numberDiff line numberDiff line change
@@ -412,10 +412,6 @@ stages:
412412
name: 'onnxruntime-extensions-Windows-CPU'
413413

414414
steps:
415-
# the vcpkg build requires a cmake python module
416-
- script: python -m pip install cmake
417-
displayName: Install cmake python module
418-
419415
- script: |
420416
call .\build.bat -DOCOS_ENABLE_CTEST=ON -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded
421417
cd out/Windows

‎CMakeLists.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -726,7 +726,7 @@ target_link_libraries(ocos_operators PRIVATE ${ocos_libraries})
726726

727727
set (file_patterns "shared/lib/*.cc")
728728
if (OCOS_ENABLE_C_API)
729-
list(APPEND file_patterns "shared/api/*.h*" "shared/api/*.cc")
729+
list(APPEND file_patterns "shared/api/*.h*" "shared/api/*.c" "shared/api/*.cc")
730730
endif()
731731

732732
file(GLOB shared_TARGET_LIB_SRC ${file_patterns})

‎cmake/ext_python.cmake

+4-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,10 @@ if (NOT Python3_FOUND)
1010
message(FATAL_ERROR "Python3 not found!")
1111
endif()
1212

13-
file(GLOB TARGET_SRC_PYOPS "pyop/*.cc" "pyop/*.h" "shared/*.cc")
13+
file(GLOB TARGET_SRC_PYOPS "pyop/pyfunc.cc" "pyop/*.h" "shared/*.cc")
14+
if (OCOS_ENABLE_C_API)
15+
list(APPEND TARGET_SRC_PYOPS "pyop/py_c_api.cc")
16+
endif()
1417
if (WIN32)
1518
list(APPEND TARGET_SRC_PYOPS "pyop/extensions_pydll.def")
1619
endif()

‎cmake/externals/opencv-no-rtti.patch

+37-16
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,17 @@
1-
diff --git a/3rdparty/libjpeg-turbo/CMakeLists.txt b/3rdparty/libjpeg-turbo/CMakeLists.txt
2-
index 3c7f29b08e..066ea4e545 100644
3-
--- a/3rdparty/libjpeg-turbo/CMakeLists.txt
4-
+++ b/3rdparty/libjpeg-turbo/CMakeLists.txt
5-
@@ -67,7 +67,7 @@ set(JPEG_LIB_VERSION "${VERSION}-${JPEG_LIB_VERSION}" PARENT_SCOPE)
6-
set(THREAD_LOCAL "") # WITH_TURBOJPEG is not used
1+
diff --git a/3rdparty/libjpeg/CMakeLists.txt b/3rdparty/libjpeg/CMakeLists.txt
2+
index c0524cc38..69a71e416 100644
3+
--- a/3rdparty/libjpeg/CMakeLists.txt
4+
+++ b/3rdparty/libjpeg/CMakeLists.txt
5+
@@ -27,7 +27,6 @@ endif()
76

8-
if(MSVC)
9-
- add_definitions(-W3 -wd4996 -wd4018)
10-
+ add_definitions(-W3)
11-
endif()
7+
ocv_warnings_disable(CMAKE_C_FLAGS -Wcast-align -Wshadow -Wunused -Wshift-negative-value -Wimplicit-fallthrough)
8+
ocv_warnings_disable(CMAKE_C_FLAGS -Wunused-parameter) # clang
9+
-ocv_warnings_disable(CMAKE_C_FLAGS /wd4013 /wd4244 /wd4267) # vs2005
1210

13-
if(WIN32)
11+
set_target_properties(${JPEG_LIBRARY}
12+
PROPERTIES OUTPUT_NAME ${JPEG_LIBRARY}
1413
diff --git a/3rdparty/zlib/CMakeLists.txt b/3rdparty/zlib/CMakeLists.txt
15-
index 9758861a6b..9e654ba922 100644
14+
index 9758861a6..9e654ba92 100644
1615
--- a/3rdparty/zlib/CMakeLists.txt
1716
+++ b/3rdparty/zlib/CMakeLists.txt
1817
@@ -81,7 +81,6 @@ set_target_properties(${ZLIB_LIBRARY} PROPERTIES DEFINE_SYMBOL ZLIB_DLL)
@@ -24,7 +23,7 @@ index 9758861a6b..9e654ba922 100644
2423
)
2524

2625
diff --git a/CMakeLists.txt b/CMakeLists.txt
27-
index d95e5db163..db185453df 100644
26+
index d95e5db16..db185453d 100644
2827
--- a/CMakeLists.txt
2928
+++ b/CMakeLists.txt
3029
@@ -617,11 +617,6 @@ endif()
@@ -40,7 +39,7 @@ index d95e5db163..db185453df 100644
4039

4140
ocv_cmake_hook(POST_COMPILER_OPTIONS)
4241
diff --git a/cmake/OpenCVDetectCXXCompiler.cmake b/cmake/OpenCVDetectCXXCompiler.cmake
43-
index 7f229cde96..92e204a5b9 100644
42+
index 7f229cde9..92e204a5b 100644
4443
--- a/cmake/OpenCVDetectCXXCompiler.cmake
4544
+++ b/cmake/OpenCVDetectCXXCompiler.cmake
4645
@@ -171,7 +171,7 @@ elseif(MSVC)
@@ -53,7 +52,7 @@ index 7f229cde96..92e204a5b9 100644
5352
else()
5453
message(WARNING "OpenCV does not recognize MSVC_VERSION \"${MSVC_VERSION}\". Cannot set OpenCV_RUNTIME")
5554
diff --git a/modules/core/include/opencv2/core/ocl.hpp b/modules/core/include/opencv2/core/ocl.hpp
56-
index 4503fa00dd..642b0508d0 100644
55+
index 4503fa00d..642b0508d 100644
5756
--- a/modules/core/include/opencv2/core/ocl.hpp
5857
+++ b/modules/core/include/opencv2/core/ocl.hpp
5958
@@ -302,21 +302,6 @@ public:
@@ -79,7 +78,7 @@ index 4503fa00dd..642b0508d0 100644
7978
inline Impl* getImpl() const { return (Impl*)p; }
8079
inline bool empty() const { return !p; }
8180
diff --git a/modules/core/src/ocl_disabled.impl.hpp b/modules/core/src/ocl_disabled.impl.hpp
82-
index a217979a1e..0ba30d024c 100644
81+
index a217979a1..0ba30d024 100644
8382
--- a/modules/core/src/ocl_disabled.impl.hpp
8483
+++ b/modules/core/src/ocl_disabled.impl.hpp
8584
@@ -177,11 +177,6 @@ void* Context::getOpenCLContextProperty(int /*propertyId*/) const { OCL_NOT_AVAI
@@ -94,3 +93,25 @@ index a217979a1e..0ba30d024c 100644
9493
/* static */ Context Context::fromHandle(void* context) { OCL_NOT_AVAILABLE(); }
9594
/* static */ Context Context::fromDevice(const ocl::Device& device) { OCL_NOT_AVAILABLE(); }
9695
/* static */ Context Context::create(const std::string& configuration) { OCL_NOT_AVAILABLE(); }
96+
diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt b/samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt
97+
deleted file mode 100644
98+
index 6887c2ab2..000000000
99+
--- a/samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt
100+
+++ /dev/null
101+
@@ -1,15 +0,0 @@
102+
-# Python 3.7.5
103+
-onnx>=1.7.0
104+
-numpy>=1.19.1
105+
-
106+
-torch>=1.5.1
107+
-torchvision>=0.6.1
108+
-
109+
-tensorflow>=2.1.0
110+
-tensorflow-gpu>=2.1.0
111+
-
112+
-paddlepaddle>=2.0.0
113+
-paddlepaddle-gpu>=2.0.0
114+
-paddlehub>=2.1.0
115+
-paddle2onnx>=0.5.1
116+
-paddleseg>=2.0.0
117+
\ No newline at end of file

‎docs/c_api.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,4 @@ Most APIs accept raw data inputs such as audio, image compressed binary formats,
1616

1717
**Image processing:** `OrtxCreateProcessor` can create an image processor object from a pre-defined workflow in JSON format to process image files into a tensor-like data type. An example code snippet can be found [here](../test/pp_api_test/test_processor.cc#L75).
1818

19-
**Audio feature extraction:** `OrtxCreateSpeechFeatureExtractor` creates a speech feature extractor to obtain log mel spectrum data as input for the Whisper model. An example code snippet can be found [here](../test/pp_api_test/test_feature_extractor.cc#L16).
19+
**Audio feature extraction:** `OrtxCreateSpeechFeatureExtractor` creates a speech feature extractor to obtain log mel spectrum data as input for the Whisper model. An example code snippet can be found [here](../test/pp_api_test/test_feature_extraction.cc#L16).

‎include/ortx_types.h

+20
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,23 @@ typedef enum {
3333
kOrtxErrorInternal = 9,
3434
kOrtxErrorUnknown = 1000
3535
} extError_t;
36+
37+
typedef enum {
38+
kOrtxUnknownType = 0,
39+
kOrtxFloat = 1,
40+
kOrtxDouble = 2,
41+
kOrtxString = 3,
42+
kOrtxBool = 4,
43+
kOrtxComplex64 = 5,
44+
kOrtxComplex128 = 6,
45+
kOrtxBFloat16 = 7,
46+
kOrtxUint8 = 8,
47+
kOrtxInt8 = 9,
48+
kOrtxUint16 = 10,
49+
kOrtxInt16 = 11,
50+
kOrtxInt32 = 12,
51+
kOrtxUint32 = 13,
52+
kOrtxInt64 = 14,
53+
kOrtxUint64 = 15,
54+
kOrtxFloat16 = 16
55+
} extDataType_t;

‎include/ortx_utils.h

+13
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,19 @@ extError_t ORTX_API_CALL OrtxDisposeOnly(OrtxObject* object);
9292
*/
9393
extError_t ORTX_API_CALL OrtxTensorResultGetAt(OrtxTensorResult* result, size_t index, OrtxTensor** tensor);
9494

95+
/**
96+
* @brief Retrieves the data type of the given tensor.
97+
*
98+
* This function returns the data type of the specified tensor. The data type is
99+
* stored in the `type` parameter.
100+
*
101+
* @param tensor The tensor for which to retrieve the data type.
102+
* @param type A pointer to a variable that will hold the retrieved data type.
103+
*
104+
* @return An `extError_t` value indicating the success or failure of the operation.
105+
*/
106+
extError_t ORTX_API_CALL OrtxGetTensorType(OrtxTensor* tensor, extDataType_t* type);
107+
95108
/** \brief Get the data from the tensor
96109
*
97110
* \param tensor The tensor object

‎onnxruntime_extensions/pp_api.py

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Copyright (c) Microsoft Corporation. All rights reserved.
2+
# Licensed under the MIT License. See License.txt in the project root for
3+
# license information.
4+
###############################################################################
5+
6+
from . import _extensions_pydll as _C
7+
if not hasattr(_C, "create_processor"):
8+
raise ImportError("onnxruntime_extensions is not built with pre-processing API")
9+
10+
create_processor = _C.create_processor
11+
load_images = _C.load_images
12+
image_pre_process = _C.image_pre_process
13+
tensor_result_get_at = _C.tensor_result_get_at

‎pyop/py_c_api.cc

+114
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
#include <pybind11/iostream.h>
5+
#include <pybind11/pybind11.h>
6+
#include <pybind11/stl.h>
7+
#include <pybind11/functional.h>
8+
#include <pybind11/numpy.h>
9+
#include <thread>
10+
11+
#include "ortx_utils.h"
12+
#include "ortx_processor.h"
13+
#include "pykernel.h"
14+
15+
namespace py = pybind11;
16+
17+
template <typename T>
18+
int64_t NumOfElement(const T& sp) {
19+
size_t c = 1;
20+
for (auto v : sp) {
21+
c *= v;
22+
}
23+
return c;
24+
}
25+
26+
void AddGlobalMethodsCApi(pybind11::module& m) {
27+
m.def(
28+
"create_processor",
29+
[](const char* processor_def_json) {
30+
OrtxProcessor* processor = nullptr;
31+
auto err = OrtxCreateProcessor(&processor, processor_def_json);
32+
if (err != kOrtxOK) {
33+
throw std::runtime_error(std::string("Failed to create processor") + OrtxGetLastErrorMessage());
34+
}
35+
return reinterpret_cast<std::uintptr_t>(processor);
36+
},
37+
"Create a processor.");
38+
39+
m.def(
40+
"load_images",
41+
[](const std::vector<std::string>& image_paths) {
42+
OrtxRawImages* images = nullptr;
43+
size_t num_images = image_paths.size();
44+
auto image_ptrs = std::make_unique<const char*[]>(num_images);
45+
for (size_t i = 0; i < num_images; ++i) {
46+
image_ptrs[i] = image_paths[i].c_str();
47+
}
48+
49+
auto err = OrtxLoadImages(&images, image_ptrs.get(), num_images, nullptr);
50+
if (err != kOrtxOK) {
51+
throw std::runtime_error(std::string("Failed to load images") + OrtxGetLastErrorMessage());
52+
}
53+
return reinterpret_cast<std::uintptr_t>(images);
54+
},
55+
"Load images.");
56+
57+
m.def(
58+
"image_pre_process",
59+
[](std::uintptr_t processor_h, std::uintptr_t images_h) {
60+
OrtxProcessor* processor = reinterpret_cast<OrtxProcessor*>(processor_h);
61+
OrtxRawImages* images = reinterpret_cast<OrtxRawImages*>(images_h);
62+
OrtxTensorResult* result{};
63+
auto err = OrtxImagePreProcess(processor, images, &result);
64+
if (err != kOrtxOK) {
65+
throw std::runtime_error(std::string("Failed to preprocess images") + OrtxGetLastErrorMessage());
66+
}
67+
return reinterpret_cast<std::uintptr_t>(result);
68+
},
69+
"Preprocess images.");
70+
71+
m.def("tensor_result_get_at", [](std::uintptr_t result_h, size_t index) {
72+
OrtxTensorResult* result = reinterpret_cast<OrtxTensorResult*>(result_h);
73+
OrtxTensor* tensor{};
74+
auto err = OrtxTensorResultGetAt(result, index, &tensor);
75+
if (err != kOrtxOK) {
76+
throw std::runtime_error(std::string("Failed to get tensor") + OrtxGetLastErrorMessage());
77+
}
78+
79+
extDataType_t tensor_type;
80+
81+
OrtxGetTensorType(tensor, &tensor_type);
82+
const int64_t* shape{};
83+
size_t num_dims;
84+
const void* data{};
85+
size_t elem_size = 0;
86+
if (tensor_type == extDataType_t::kOrtxInt64 || tensor_type == extDataType_t::kOrtxFloat) {
87+
OrtxGetTensorData(tensor, reinterpret_cast<const void**>(&data), &shape, &num_dims);
88+
elem_size = 4;
89+
if (tensor_type == extDataType_t::kOrtxInt64) {
90+
elem_size = 8;
91+
}
92+
} else if (tensor_type == extDataType_t::kOrtxUnknownType) {
93+
throw std::runtime_error("Failed to get tensor type");
94+
} else if (tensor_type == extDataType_t::kOrtxUnknownType) {
95+
throw std::runtime_error("unsupported tensor type");
96+
}
97+
98+
std::vector<std::size_t> npy_dims;
99+
for (auto n = num_dims - num_dims; n < num_dims; ++n) {
100+
npy_dims.push_back(shape[n]);
101+
}
102+
py::array obj{};
103+
104+
if (tensor_type == extDataType_t::kOrtxFloat) {
105+
obj = py::array_t<float>(npy_dims);
106+
} else if (tensor_type == extDataType_t::kOrtxInt64) {
107+
obj = py::array_t<int64_t>(npy_dims);
108+
}
109+
110+
void* out_ptr = obj.mutable_data();
111+
memcpy(out_ptr, data, NumOfElement(npy_dims) * elem_size);
112+
return obj;
113+
}, "Get tensor at index.");
114+
}

‎pyop/pyfunc.cc

+3
Original file line numberDiff line numberDiff line change
@@ -482,6 +482,9 @@ PYBIND11_MODULE(_extensions_pydll, m) {
482482
m.doc() = "pybind11 stateful interface to ONNXRuntime-Extensions";
483483

484484
AddGlobalMethods(m);
485+
#if defined(ENABLE_C_API)
486+
AddGlobalMethodsCApi(m);
487+
#endif
485488
AddObjectMethods(m);
486489
auto atexit = py::module_::import("atexit");
487490
atexit.attr("register")(py::cpp_function([]() {

‎pyop/pykernel.h

+4
Original file line numberDiff line numberDiff line change
@@ -127,3 +127,7 @@ struct PyCustomOpFactory : public OrtCustomOp {
127127
};
128128

129129
bool EnablePyCustomOps(bool enable = true);
130+
131+
#if defined(ENABLE_C_API)
132+
void AddGlobalMethodsCApi(pybind11::module& m);
133+
#endif

‎shared/api/c_api_utils.cc

+23-6
Original file line numberDiff line numberDiff line change
@@ -101,28 +101,45 @@ extError_t ORTX_API_CALL OrtxTensorResultGetAt(OrtxTensorResult* result, size_t
101101
return kOrtxErrorInvalidArgument;
102102
}
103103

104-
auto tensor_ptr = std::make_unique<OrtxObjectWrapper<ortc::TensorBase, kOrtxKindTensor>>();
105-
tensor_ptr->SetObject(ts);
104+
auto tensor_ptr = std::make_unique<TensorObject>();
105+
tensor_ptr->SetTensor(ts);
106+
tensor_ptr->SetTensorType(result_ptr->GetTensorType(index));
106107
*tensor = static_cast<OrtxTensor*>(tensor_ptr.release());
107108
return extError_t();
108109
}
109110

111+
extError_t ORTX_API_CALL OrtxGetTensorType(OrtxTensor* tensor, extDataType_t* type) {
112+
if (tensor == nullptr || type == nullptr) {
113+
ReturnableStatus::last_error_message_ = "Invalid argument";
114+
return kOrtxErrorInvalidArgument;
115+
}
116+
117+
auto tensor_impl = static_cast<TensorObject*>(tensor);
118+
if (tensor_impl->ortx_kind() != extObjectKind_t::kOrtxKindTensor) {
119+
ReturnableStatus::last_error_message_ = "Invalid argument";
120+
return kOrtxErrorInvalidArgument;
121+
}
122+
123+
*type = tensor_impl->GetTensorType();
124+
return extError_t();
125+
}
126+
110127
extError_t ORTX_API_CALL OrtxGetTensorData(OrtxTensor* tensor, const void** data, const int64_t** shape,
111128
size_t* num_dims) {
112129
if (tensor == nullptr) {
113130
ReturnableStatus::last_error_message_ = "Invalid argument";
114131
return kOrtxErrorInvalidArgument;
115132
}
116133

117-
auto tensor_impl = static_cast<OrtxObjectWrapper<ortc::TensorBase, kOrtxKindTensor>*>(tensor);
134+
auto tensor_impl = static_cast<TensorObject*>(tensor);
118135
if (tensor_impl->ortx_kind() != extObjectKind_t::kOrtxKindTensor) {
119136
ReturnableStatus::last_error_message_ = "Invalid argument";
120137
return kOrtxErrorInvalidArgument;
121138
}
122139

123-
*data = tensor_impl->GetObject()->DataRaw();
124-
*shape = tensor_impl->GetObject()->Shape().data();
125-
*num_dims = tensor_impl->GetObject()->Shape().size();
140+
*data = tensor_impl->GetTensor()->DataRaw();
141+
*shape = tensor_impl->GetTensor()->Shape().data();
142+
*num_dims = tensor_impl->GetTensor()->Shape().size();
126143
return extError_t();
127144
}
128145

0 commit comments

Comments
 (0)