From 8fba531b8eb9ebe1bd09ed76ad4dfd4d1b199ba7 Mon Sep 17 00:00:00 2001 From: Yueqing Zhang Date: Tue, 12 Nov 2024 21:15:49 -0600 Subject: [PATCH 1/4] add support for error throwing --- .../core/providers/vitisai/imp/global_api.cc | 30 +++++++++++++++---- .../providers/vitisai/include/vaip/my_ort.h | 1 + 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/onnxruntime/core/providers/vitisai/imp/global_api.cc b/onnxruntime/core/providers/vitisai/imp/global_api.cc index 3e802e5a77203..5e3ad7a468e2b 100644 --- a/onnxruntime/core/providers/vitisai/imp/global_api.cc +++ b/onnxruntime/core/providers/vitisai/imp/global_api.cc @@ -47,6 +47,8 @@ struct OrtVitisAIEpAPI { void (*initialize_onnxruntime_vitisai_ep)(vaip_core::OrtApiForVaip* api, std::vector& ret_domain); std::vector>* (*compile_onnx_model_with_options)( const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options); + std::vector>* (*compile_onnx_model_vitisai_ep_with_error_handling)( + const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options, void* status, vaip_core::error_report_func func); uint32_t (*vaip_get_version)(); void (*create_ep_context_nodes)( const std::vector>& eps, @@ -74,10 +76,11 @@ struct OrtVitisAIEpAPI { ORT_THROW_IF_ERROR(env.LoadDynamicLibrary(full_path, true, &handle_)); #endif ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "initialize_onnxruntime_vitisai_ep", (void**)&initialize_onnxruntime_vitisai_ep)); - auto status = env.GetSymbolFromLibrary(handle_, "compile_onnx_model_vitisai_ep_with_options", (void**)&compile_onnx_model_with_options); - if (!status.IsOK()) { - ::onnxruntime::LogRuntimeError(0, status, __FILE__, static_cast(__FUNCTION__), __LINE__); - ORT_THROW(status); + auto status1 = env.GetSymbolFromLibrary(handle_, "compile_onnx_model_vitisai_ep_with_error_handling", (void**)&compile_onnx_model_vitisai_ep_with_error_handling); + auto status2 = env.GetSymbolFromLibrary(handle_, "compile_onnx_model_vitisai_ep_with_options", (void**)&compile_onnx_model_with_options); + if ((!status1.IsOK()) && (!status2.IsOK())) { + ::onnxruntime::LogRuntimeError(0, status2, __FILE__, static_cast(__FUNCTION__), __LINE__); + ORT_THROW(status2); } std::ignore = env.GetSymbolFromLibrary(handle_, "vaip_get_version", (void**)&vaip_get_version); @@ -97,10 +100,25 @@ static vaip_core::OrtApiForVaip the_global_api; std::shared_ptr get_kernel_registry_vitisaiep() { return s_kernel_registry_vitisaiep; } const std::vector& get_domains_vitisaiep() { return s_domains_vitisaiep; } +void change_status_with_error(void* status_ptr, int error_code, const char* error_msg) { + auto status = reinterpret_cast(status_ptr); + *status = Status(onnxruntime::common::ONNXRUNTIME, error_code, error_msg); +} + vaip_core::DllSafe>> compile_onnx_model( - const onnxruntime::GraphViewer& graph_viewer, const logging::Logger& logger, const ProviderOptions& options) { + const onnxruntime::GraphViewer& graph_viewer, const onnxruntime::logging::Logger& logger, const onnxruntime::ProviderOptions& options) { auto model_path = graph_viewer.ModelPath().string(); - return vaip_core::DllSafe(s_library_vitisaiep.compile_onnx_model_with_options(model_path, graph_viewer.GetGraph(), options)); + if (s_library_vitisaiep.compile_onnx_model_vitisai_ep_with_error_handling) { + Status status = Status::OK(); + auto status_ptr = reinterpret_cast(&status); + auto ret = vaip_core::DllSafe(s_library_vitisaiep.compile_onnx_model_vitisai_ep_with_error_handling(model_path, graph_viewer.GetGraph(), options, status_ptr, change_status_with_error)); + if (!status.IsOK()) { + ORT_THROW(status); + } + return ret; + } else { + return vaip_core::DllSafe(s_library_vitisaiep.compile_onnx_model_with_options(model_path, graph_viewer.GetGraph(), options)); + } } std::optional> create_ep_context_nodes( diff --git a/onnxruntime/core/providers/vitisai/include/vaip/my_ort.h b/onnxruntime/core/providers/vitisai/include/vaip/my_ort.h index 7628e45d2b933..85a1262d8489b 100644 --- a/onnxruntime/core/providers/vitisai/include/vaip/my_ort.h +++ b/onnxruntime/core/providers/vitisai/include/vaip/my_ort.h @@ -122,4 +122,5 @@ using InitializedTensorSet = std::unordered_map; using ModelMetaData = std::unordered_map; +using error_report_func = void (*)(void*, int, const char*); } // namespace vaip_core From 3e124671b165bf7237d190c57d2cb90e5afb4475 Mon Sep 17 00:00:00 2001 From: Yueqing Zhang Date: Tue, 5 Nov 2024 05:02:06 -0600 Subject: [PATCH 2/4] add profiling api for vitis ai ep --- .../core/providers/shared_library/provider_interfaces.h | 1 + .../core/providers/shared_library/provider_wrappedtypes.h | 3 +++ onnxruntime/core/providers/vitisai/imp/global_api.cc | 4 ++++ .../core/providers/vitisai/include/vaip/vaip_ort_api.h | 3 ++- onnxruntime/core/session/provider_bridge_ort.cc | 1 + 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index 1436afa41c2f8..aeb2ec2e96960 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -585,6 +585,7 @@ struct ProviderHost { virtual const ConfigOptions& RunOptions__GetConfigOptions(const RunOptions* p) = 0; // OrtSessionOptions virtual const std::unordered_map& SessionOptions__GetConfigOptionsMap(const OrtSessionOptions* p) = 0; + virtual bool SessionOptions__GetEnableProfiling(const OrtSessionOptions* p) = 0; // ComputeCapability virtual std::unique_ptr ComputeCapability__construct(std::unique_ptr t_sub_graph) = 0; virtual void ComputeCapability__operator_delete(ComputeCapability* p) = 0; diff --git a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h index 5e8996d590db8..1d9ad2a4be776 100644 --- a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h +++ b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h @@ -1466,5 +1466,8 @@ struct OrtSessionOptions final { const std::unordered_map& GetConfigOptions() const { return onnxruntime::g_host->SessionOptions__GetConfigOptionsMap(this); } + bool GetEnableProfiling() const { + return onnxruntime::g_host->SessionOptions__GetEnableProfiling(this); + } PROVIDER_DISALLOW_ALL(OrtSessionOptions) }; diff --git a/onnxruntime/core/providers/vitisai/imp/global_api.cc b/onnxruntime/core/providers/vitisai/imp/global_api.cc index 5e3ad7a468e2b..d61c8bce4c172 100644 --- a/onnxruntime/core/providers/vitisai/imp/global_api.cc +++ b/onnxruntime/core/providers/vitisai/imp/global_api.cc @@ -467,6 +467,10 @@ vaip_core::OrtApiForVaip* create_org_api_hook() { return vaip_core::DllSafe(model_proto.SerializeAsString()); }; the_global_api.model_proto_delete = [](ONNX_NAMESPACE::ModelProto* p) { delete p; }; + the_global_api.is_profiling_enabled = [](void* session_options) { + auto options = reinterpret_cast(session_options); + return options->GetEnableProfiling(); + }; if (!s_library_vitisaiep.vaip_get_version) { return reinterpret_cast(&(the_global_api.host_)); } else { diff --git a/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h b/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h index 288cfd6850d06..7ca46cefe0d91 100644 --- a/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h +++ b/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h @@ -13,7 +13,7 @@ struct OrtApi; namespace vaip_core { -#define VAIP_ORT_API_MAJOR (11u) +#define VAIP_ORT_API_MAJOR (12u) #define VAIP_ORT_API_MINOR (0u) #define VAIP_ORT_API_PATCH (0u) struct OrtApiForVaip { @@ -234,6 +234,7 @@ struct OrtApiForVaip { ModelProto* (*model_to_proto)(Model& model); // [95] DllSafe (*model_proto_serialize_as_string)(ModelProto& model_proto); // [96] void (*model_proto_delete)(ModelProto* p); // [97] + bool (*is_profiling_enabled)(void* session_options); // [98] }; #ifndef USE_VITISAI diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index ecbdd31160c7a..9a76db80eecff 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -718,6 +718,7 @@ struct ProviderHostImpl : ProviderHost { // OrtSessionOptions (wrapped) const std::unordered_map& SessionOptions__GetConfigOptionsMap(const OrtSessionOptions* p) override { return p->value.config_options.configurations; } + bool SessionOptions__GetEnableProfiling(const OrtSessionOptions* p) override { return p->value.enable_profiling; }; // ComputeCapability (wrapped) std::unique_ptr ComputeCapability__construct(std::unique_ptr t_sub_graph) override { return std::make_unique(std::move(t_sub_graph)); } void ComputeCapability__operator_delete(ComputeCapability* p) override { delete p; } From 0380565f53c88f831e6c54fac79842f7b9689d64 Mon Sep 17 00:00:00 2001 From: Yueqing Zhang Date: Wed, 13 Nov 2024 00:52:35 -0600 Subject: [PATCH 3/4] wip --- .../core/providers/vitisai/imp/global_api.cc | 2 ++ .../core/providers/vitisai/imp/tensor_proto.cc | 13 +++++++++++++ .../core/providers/vitisai/imp/tensor_proto.h | 4 ++++ .../providers/vitisai/include/vaip/vaip_ort_api.h | 6 ++++++ 4 files changed, 25 insertions(+) diff --git a/onnxruntime/core/providers/vitisai/imp/global_api.cc b/onnxruntime/core/providers/vitisai/imp/global_api.cc index d61c8bce4c172..4af01c10d637e 100644 --- a/onnxruntime/core/providers/vitisai/imp/global_api.cc +++ b/onnxruntime/core/providers/vitisai/imp/global_api.cc @@ -402,10 +402,12 @@ vaip_core::OrtApiForVaip* create_org_api_hook() { the_global_api.tensor_proto_get_shape_unsafe = vaip::tensor_proto_get_shape; the_global_api.tensor_proto_data_type = [](const ONNX_NAMESPACE::TensorProto& t) -> int { return t.data_type(); }; the_global_api.tensor_proto_delete = [](ONNX_NAMESPACE::TensorProto* tp) { delete tp; }; + the_global_api.tensor_proto_new_i4 = vaip::tensor_proto_new_i4; the_global_api.tensor_proto_new_i8 = vaip::tensor_proto_new_i8; the_global_api.tensor_proto_new_i16 = vaip::tensor_proto_new_i16; the_global_api.tensor_proto_new_i32 = vaip::tensor_proto_new_i32; the_global_api.tensor_proto_new_i64 = vaip::tensor_proto_new_i64; + the_global_api.tensor_proto_new_u4 = vaip::tensor_proto_new_u4; the_global_api.tensor_proto_new_u8 = vaip::tensor_proto_new_u8; the_global_api.tensor_proto_new_u16 = vaip::tensor_proto_new_u16; the_global_api.tensor_proto_new_u32 = vaip::tensor_proto_new_u32; diff --git a/onnxruntime/core/providers/vitisai/imp/tensor_proto.cc b/onnxruntime/core/providers/vitisai/imp/tensor_proto.cc index 872d022e85264..bb942c69003a1 100644 --- a/onnxruntime/core/providers/vitisai/imp/tensor_proto.cc +++ b/onnxruntime/core/providers/vitisai/imp/tensor_proto.cc @@ -87,6 +87,12 @@ static ONNX_NAMESPACE::TensorProto* tensor_proto_new(const std::string& name, co return tensor_proto.release(); } +ONNX_NAMESPACE::TensorProto* tensor_proto_new_i4(const std::string& name, const std::vector& shape, + const std::vector& data) { + return tensor_proto_new(name, shape, ONNX_NAMESPACE::TensorProto_DataType_INT4, + reinterpret_cast(&data[0]), data.size() * sizeof(data[0])); +} + ONNX_NAMESPACE::TensorProto* tensor_proto_new_i8(const std::string& name, const std::vector& shape, const std::vector& data) { return tensor_proto_new(name, shape, ONNX_NAMESPACE::TensorProto_DataType_INT8, @@ -108,6 +114,13 @@ ONNX_NAMESPACE::TensorProto* tensor_proto_new_i64(const std::string& name, const return tensor_proto_new(name, shape, ONNX_NAMESPACE::TensorProto_DataType_INT64, reinterpret_cast(&data[0]), data.size() * sizeof(data[0])); } + +ONNX_NAMESPACE::TensorProto* tensor_proto_new_u4(const std::string& name, const std::vector& shape, + const std::vector& data) { + return tensor_proto_new(name, shape, ONNX_NAMESPACE::TensorProto_DataType_UINT4, + reinterpret_cast(&data[0]), data.size() * sizeof(data[0])); +} + ONNX_NAMESPACE::TensorProto* tensor_proto_new_u8(const std::string& name, const std::vector& shape, const std::vector& data) { return tensor_proto_new(name, shape, ONNX_NAMESPACE::TensorProto_DataType_UINT8, diff --git a/onnxruntime/core/providers/vitisai/imp/tensor_proto.h b/onnxruntime/core/providers/vitisai/imp/tensor_proto.h index 618d9c4728e2f..73015d3411a54 100644 --- a/onnxruntime/core/providers/vitisai/imp/tensor_proto.h +++ b/onnxruntime/core/providers/vitisai/imp/tensor_proto.h @@ -9,6 +9,10 @@ namespace vaip { gsl::span tensor_proto_as_raw(const onnxruntime::Graph& graph, const ONNX_NAMESPACE::TensorProto& tensor); vaip_core::DllSafe> tensor_proto_get_shape(const ONNX_NAMESPACE::TensorProto& tensor); const std::string& tensor_proto_get_name(const ONNX_NAMESPACE::TensorProto& tensor); +ONNX_NAMESPACE::TensorProto* tensor_proto_new_i4(const std::string& name, const std::vector& shape, + const std::vector& data); +ONNX_NAMESPACE::TensorProto* tensor_proto_new_u4(const std::string& name, const std::vector& shape, + const std::vector& data); ONNX_NAMESPACE::TensorProto* tensor_proto_new_i8(const std::string& name, const std::vector& shape, const std::vector& data); ONNX_NAMESPACE::TensorProto* tensor_proto_new_u8(const std::string& name, const std::vector& shape, diff --git a/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h b/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h index 7ca46cefe0d91..7ca236d95ea64 100644 --- a/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h +++ b/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h @@ -235,6 +235,12 @@ struct OrtApiForVaip { DllSafe (*model_proto_serialize_as_string)(ModelProto& model_proto); // [96] void (*model_proto_delete)(ModelProto* p); // [97] bool (*is_profiling_enabled)(void* session_options); // [98] + TensorProto* (*tensor_proto_new_i4)(const std::string& name, + const std::vector& shape, + const std::vector& data); // [99] + TensorProto* (*tensor_proto_new_u4)(const std::string& name, + const std::vector& shape, + const std::vector& data); // [100] }; #ifndef USE_VITISAI From acf21f1b89f150eb47eceb51eff85df0155d0de4 Mon Sep 17 00:00:00 2001 From: Yueqing Zhang Date: Thu, 14 Nov 2024 00:08:21 -0600 Subject: [PATCH 4/4] wip --- onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h b/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h index 7ca236d95ea64..f92955cf0b0c0 100644 --- a/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h +++ b/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h @@ -13,7 +13,7 @@ struct OrtApi; namespace vaip_core { -#define VAIP_ORT_API_MAJOR (12u) +#define VAIP_ORT_API_MAJOR (13u) #define VAIP_ORT_API_MINOR (0u) #define VAIP_ORT_API_PATCH (0u) struct OrtApiForVaip {